<!DOCTYPE HTML>
<html lang="zh-CN">


<head>
    <meta name="baidu-site-verification" content="code-KNXLvfbWBj" />
    <meta charset="utf-8">
    <meta name="keywords" content="Spark笔记, Aunean&#39;s Blog">
    <meta name="description" content="">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
    <meta name="renderer" content="webkit|ie-stand|ie-comp">
    <meta name="mobile-web-app-capable" content="yes">
    <meta name="format-detection" content="telephone=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
    <!-- Global site tag (gtag.js) - Google Analytics -->


    <title>Spark笔记 | Aunean&#39;s Blog</title>
    <link rel="icon" type="image/png" href="/favicon.png">

    <link rel="stylesheet" type="text/css" href="/libs/awesome/css/all.css">
    <link rel="stylesheet" type="text/css" href="/libs/materialize/materialize.min.css">
    <link rel="stylesheet" type="text/css" href="/libs/aos/aos.css">
    <link rel="stylesheet" type="text/css" href="/libs/animate/animate.min.css">
    <link rel="stylesheet" type="text/css" href="/libs/lightGallery/css/lightgallery.min.css">
    <link rel="stylesheet" type="text/css" href="/css/matery.css">
    <link rel="stylesheet" type="text/css" href="/css/my.css">

    <script src="/libs/jquery/jquery.min.js"></script>

<meta name="generator" content="Hexo 5.4.0">
<style>.github-emoji { position: relative; display: inline-block; width: 1.2em; min-height: 1.2em; overflow: hidden; vertical-align: top; color: transparent; }  .github-emoji > span { position: relative; z-index: 10; }  .github-emoji img, .github-emoji .fancybox { margin: 0 !important; padding: 0 !important; border: none !important; outline: none !important; text-decoration: none !important; user-select: none !important; cursor: auto !important; }  .github-emoji img { height: 1.2em !important; width: 1.2em !important; position: absolute !important; left: 50% !important; top: 50% !important; transform: translate(-50%, -50%) !important; user-select: none !important; cursor: auto !important; } .github-emoji-fallback { color: inherit; } .github-emoji-fallback img { opacity: 0 !important; }</style>
<link rel="alternate" href="/atom.xml" title="Aunean's Blog" type="application/atom+xml">
<link rel="stylesheet" href="/css/prism-tomorrow.css" type="text/css"></head>





<body>
    <header class="navbar-fixed">
    <nav id="headNav" class="bg-color nav-transparent">
        <div id="navContainer" class="nav-wrapper container">
            <div class="brand-logo">
                <a href="/" class="waves-effect waves-light">
                    
                    <img src="/medias/logo.png" class="logo-img" alt="LOGO">
                    
                    <span class="logo-span">Aunean&#39;s Blog</span>
                </a>
            </div>
            

<a href="#" data-target="mobile-nav" class="sidenav-trigger button-collapse"><i class="fas fa-bars"></i></a>
<ul class="right nav-menu">
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/" class="waves-effect waves-light">
      
      <i class="fas fa-home" style="zoom: 0.6;"></i>
      
      <span>首页</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/tags" class="waves-effect waves-light">
      
      <i class="fas fa-tags" style="zoom: 0.6;"></i>
      
      <span>标签</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/categories" class="waves-effect waves-light">
      
      <i class="fas fa-bookmark" style="zoom: 0.6;"></i>
      
      <span>分类</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/archives" class="waves-effect waves-light">
      
      <i class="fas fa-archive" style="zoom: 0.6;"></i>
      
      <span>归档</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/about" class="waves-effect waves-light">
      
      <i class="fas fa-user-circle" style="zoom: 0.6;"></i>
      
      <span>关于</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/contact" class="waves-effect waves-light">
      
      <i class="fas fa-comments" style="zoom: 0.6;"></i>
      
      <span>留言板</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/friends" class="waves-effect waves-light">
      
      <i class="fas fa-address-book" style="zoom: 0.6;"></i>
      
      <span>友情链接</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/navigate" class="waves-effect waves-light">
      
      <i class="fas fa-location-arrow" style="zoom: 0.6;"></i>
      
      <span>导航</span>
    </a>
    
  </li>
  
  <li>
    <a href="#searchModal" class="modal-trigger waves-effect waves-light">
      <i id="searchIcon" class="fas fa-search" title="搜索" style="zoom: 0.85;"></i>
    </a>
  </li>
</ul>


<div id="mobile-nav" class="side-nav sidenav">

    <div class="mobile-head bg-color">
        
        <img src="/medias/logo.png" class="logo-img circle responsive-img">
        
        <div class="logo-name">Aunean&#39;s Blog</div>
        <div class="logo-desc">
            
            Never really desperate, only the lost of the soul.
            
        </div>
    </div>

    

    <ul class="menu-list mobile-menu-list">
        
        <li class="m-nav-item">
	  
		<a href="/" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-home"></i>
			
			首页
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/tags" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-tags"></i>
			
			标签
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/categories" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-bookmark"></i>
			
			分类
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/archives" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-archive"></i>
			
			归档
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/about" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-user-circle"></i>
			
			关于
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/contact" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-comments"></i>
			
			留言板
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/friends" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-address-book"></i>
			
			友情链接
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/navigate" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-location-arrow"></i>
			
			导航
		</a>
          
        </li>
        
        
        <li><div class="divider"></div></li>
        <li>
            <a href="https://github.com/Aunean-ls" class="waves-effect waves-light" target="_blank">
                <i class="fab fa-github-square fa-fw"></i>Fork Me
            </a>
        </li>
        
    </ul>
</div>


        </div>

        
            <style>
    .nav-transparent .github-corner {
        display: none !important;
    }

    .github-corner {
        position: absolute;
        z-index: 10;
        top: 0;
        right: 0;
        border: 0;
        transform: scale(1.1);
    }

    .github-corner svg {
        color: #0f9d58;
        fill: #fff;
        height: 64px;
        width: 64px;
    }

    .github-corner:hover .octo-arm {
        animation: a 0.56s ease-in-out;
    }

    .github-corner .octo-arm {
        animation: none;
    }

    @keyframes a {
        0%,
        to {
            transform: rotate(0);
        }
        20%,
        60% {
            transform: rotate(-25deg);
        }
        40%,
        80% {
            transform: rotate(10deg);
        }
    }
</style>

<a href="https://github.com/Aunean-ls" class="github-corner tooltipped hide-on-med-and-down" target="_blank"
   data-tooltip="Fork Me" data-position="left" data-delay="50">
    <svg viewBox="0 0 250 250" aria-hidden="true">
        <path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path>
        <path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2"
              fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path>
        <path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z"
              fill="currentColor" class="octo-body"></path>
    </svg>
</a>
        
    </nav>

</header>

    
<script src="/libs/cryptojs/crypto-js.min.js"></script>
<script>
    (function() {
        let pwd = '';
        if (pwd && pwd.length > 0) {
            if (pwd !== CryptoJS.SHA256(prompt('请输入访问本文章的密码')).toString(CryptoJS.enc.Hex)) {
                alert('密码错误，将返回主页！');
                location.href = '/';
            }
        }
    })();
</script>




<div class="bg-cover pd-header post-cover" style="background-image: url('https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/u013.webp')">
    <div class="container" style="right: 0px;left: 0px;">
        <div class="row">
            <div class="col s12 m12 l12">
                <div class="brand">
                    <h1 class="description center-align post-title">Spark笔记</h1>
                </div>
            </div>
        </div>
    </div>
</div>




<main class="post-container content">

    
    <link rel="stylesheet" href="/libs/tocbot/tocbot.css">
<style>
    #articleContent h1::before,
    #articleContent h2::before,
    #articleContent h3::before,
    #articleContent h4::before,
    #articleContent h5::before,
    #articleContent h6::before {
        display: block;
        content: " ";
        height: 100px;
        margin-top: -100px;
        visibility: hidden;
    }

    #articleContent :focus {
        outline: none;
    }

    .toc-fixed {
        position: fixed;
        top: 64px;
    }

    .toc-widget {
        width: 345px;
        padding-left: 20px;
    }

    .toc-widget .toc-title {
        padding: 35px 0 15px 17px;
        font-size: 1.5rem;
        font-weight: bold;
        line-height: 1.5rem;
    }

    .toc-widget ol {
        padding: 0;
        list-style: none;
    }

    #toc-content {
        padding-bottom: 30px;
        overflow: auto;
    }

    #toc-content ol {
        padding-left: 10px;
    }

    #toc-content ol li {
        padding-left: 10px;
    }

    #toc-content .toc-link:hover {
        color: #42b983;
        font-weight: 700;
        text-decoration: underline;
    }

    #toc-content .toc-link::before {
        background-color: transparent;
        max-height: 25px;

        position: absolute;
        right: 23.5vw;
        display: block;
    }

    #toc-content .is-active-link {
        color: #42b983;
    }

    #floating-toc-btn {
        position: fixed;
        right: 15px;
        bottom: 76px;
        padding-top: 15px;
        margin-bottom: 0;
        z-index: 998;
    }

    #floating-toc-btn .btn-floating {
        width: 48px;
        height: 48px;
    }

    #floating-toc-btn .btn-floating i {
        line-height: 48px;
        font-size: 1.4rem;
    }
</style>
<div class="row">
    <div id="main-content" class="col s12 m12 l9">
        <!-- 文章内容详情 -->
<div id="artDetail">
    <div class="card">
        <div class="card-content article-info">
            <div class="row tag-cate">
                <div class="col s7">
                    
                    <div class="article-tag">
                        
                            <a href="/tags/Spark/">
                                <span class="chip bg-color">Spark</span>
                            </a>
                        
                    </div>
                    
                </div>
                <div class="col s5 right-align">
                    
                    <div class="post-cate">
                        <i class="fas fa-bookmark fa-fw icon-category"></i>
                        
                            <a href="/categories/%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/" class="post-category">
                                学习笔记
                            </a>
                        
                    </div>
                    
                </div>
            </div>

            <div class="post-info">
                
                <div class="post-date info-break-policy">
                    <i class="far fa-calendar-minus fa-fw"></i>发布日期:&nbsp;&nbsp;
                    2021-05-09
                </div>
                

                
                <div class="post-date info-break-policy">
                    <i class="far fa-calendar-check fa-fw"></i>更新日期:&nbsp;&nbsp;
                    2021-08-04
                </div>
                

                
                <div class="info-break-policy">
                    <i class="far fa-file-word fa-fw"></i>文章字数:&nbsp;&nbsp;
                    8k
                </div>
                

                

                
            </div>
        </div>
        <hr class="clearfix">

        

        

        <div class="card-content article-card-content">
            <div id="articleContent">
                <pre class=" language-shell"><code class="language-shell"># 第一种方式
bin/spark-submit --master spark://node:7077 --class cn.hnkjxy.WordCount_Online /root/data/spark5.jar

# 第二种方式
bin/spark-submit --master spark://node:7077 --class cn.hnkjxy.WordCount_Online /root/data/spark5.jar /hdfs路劲

# 从本地文件系统加载数据创建RDD
val localRDD=sc.textFile("file:///root/words.txt")

# 从hdfs加载数据创建RDD
val hdfsRDD = sc.textFile("/spark/test/words.txt")

# 通过集合创建RDD
scala> val arr = Array(1,2,3,4,5)
scala> val arrRDD = sc.parallelize(arr)

# 通过列表创建RDD
val arr1 = List(1, 2, 3, 4, 5, 6, 7)
val arr1RDD = sc.parallelize(arr1)

</code></pre>
<h2 id="RDD的分区"><a href="#RDD的分区" class="headerlink" title="RDD的分区"></a>RDD的分区</h2><ul>
<li>哈希分区（HashPartitioner）</li>
<li>范围分区（RangePartitioner）</li>
</ul>
<h2 id="RDD的依赖关系"><a href="#RDD的依赖关系" class="headerlink" title="RDD的依赖关系"></a>RDD的依赖关系</h2><ul>
<li><p>窄依赖</p>
<ul>
<li>窄依赖是指父RDD的每一个分区最多被一个子RDD的分区使用</li>
<li>RDD做map、filter和union算子操作时，是属于窄依赖的第一类表现；而RDD做join算子操作（对输入进行协同划分）时，是属于窄依赖表现的第二类。</li>
</ul>
</li>
<li><p>宽依赖</p>
<ul>
<li><p>宽依赖是指子RDD的每一个分区都会使用所有父RDD的所有分区或多个分区</p>
</li>
<li></li>
</ul>
</li>
</ul>
<h2 id="RDD的机制"><a href="#RDD的机制" class="headerlink" title="RDD的机制"></a>RDD的机制</h2><h3 id="持久化机制"><a href="#持久化机制" class="headerlink" title="持久化机制"></a>持久化机制</h3><ol>
<li>使用persist()方法对RDD进行持久化</li>
</ol>
<pre class=" language-scala"><code class="language-scala"><span class="token comment" spellcheck="true">// 导入StorageLevel对象的包</span>
scala<span class="token operator">></span> <span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>storage<span class="token punctuation">.</span>StorageLevel
<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>storage<span class="token punctuation">.</span>StorageLevel

<span class="token comment" spellcheck="true">// 定义一个列表list</span>
scala<span class="token operator">></span> <span class="token keyword">val</span> list <span class="token operator">=</span> List<span class="token punctuation">(</span><span class="token string">"hadoop"</span><span class="token punctuation">,</span><span class="token string">"spark"</span><span class="token punctuation">,</span><span class="token string">"hive"</span><span class="token punctuation">)</span>
list<span class="token operator">:</span> List<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span> <span class="token operator">=</span> List<span class="token punctuation">(</span>hadoop<span class="token punctuation">,</span> spark<span class="token punctuation">,</span> hive<span class="token punctuation">)</span>

<span class="token comment" spellcheck="true">// 创建一个RDD</span>
scala<span class="token operator">></span> <span class="token keyword">val</span> listRDD<span class="token operator">=</span>sc<span class="token punctuation">.</span>parallelize<span class="token punctuation">(</span>list<span class="token punctuation">)</span>
listRDD<span class="token operator">:</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>RDD<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span> <span class="token operator">=</span> ParallelCollectionRDD<span class="token punctuation">[</span><span class="token number">0</span><span class="token punctuation">]</span> at parallelize at <span class="token operator">&lt;</span>console<span class="token operator">></span><span class="token operator">:</span><span class="token number">27</span>

<span class="token comment" spellcheck="true">// 添加persist()方法，用于持久化RDD，减少IO操作，提高计算效率</span>
scala<span class="token operator">></span> listRDD<span class="token punctuation">.</span>persist<span class="token punctuation">(</span>StorageLevel<span class="token punctuation">.</span>DISK_ONLY<span class="token punctuation">)</span>
res0<span class="token operator">:</span> listRDD<span class="token punctuation">.</span><span class="token keyword">type</span> <span class="token operator">=</span> ParallelCollectionRDD<span class="token punctuation">[</span><span class="token number">0</span><span class="token punctuation">]</span> at parallelize at <span class="token operator">&lt;</span>console<span class="token operator">></span><span class="token operator">:</span><span class="token number">27</span>

<span class="token comment" spellcheck="true">// count()行动算子操作，统计元素的个数</span>
scala<span class="token operator">></span> println<span class="token punctuation">(</span>listRDD<span class="token punctuation">.</span>count<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
<span class="token number">3</span>                                                                               
scala<span class="token operator">></span> println<span class="token punctuation">(</span>listRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>mkString<span class="token punctuation">(</span><span class="token string">""</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
hadoopsparkhive

scala<span class="token operator">></span> println<span class="token punctuation">(</span>listRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>mkString<span class="token punctuation">(</span><span class="token string">","</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
hadoop<span class="token punctuation">,</span>spark<span class="token punctuation">,</span>hive

scala<span class="token operator">></span> listRDD<span class="token punctuation">.</span>unpersist<span class="token punctuation">(</span><span class="token punctuation">)</span>
res3<span class="token operator">:</span> listRDD<span class="token punctuation">.</span><span class="token keyword">type</span> <span class="token operator">=</span> ParallelCollectionRDD<span class="token punctuation">[</span><span class="token number">0</span><span class="token punctuation">]</span> at parallelize at <span class="token operator">&lt;</span>console<span class="token operator">></span><span class="token operator">:</span><span class="token number">27</span>
</code></pre>
<ol start="2">
<li>使用chache()方法对RDD进行持久化 </li>
</ol>
<pre class=" language-scala"><code class="language-scala">scala<span class="token operator">></span> <span class="token keyword">val</span> list <span class="token operator">=</span> List<span class="token punctuation">(</span><span class="token string">"hadoop"</span><span class="token punctuation">,</span><span class="token string">"spark"</span><span class="token punctuation">,</span><span class="token string">"hive"</span><span class="token punctuation">)</span>
list<span class="token operator">:</span> List<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span> <span class="token operator">=</span> List<span class="token punctuation">(</span>hadoop<span class="token punctuation">,</span> spark<span class="token punctuation">,</span> hive<span class="token punctuation">)</span>

scala<span class="token operator">></span> listRDD<span class="token punctuation">.</span>cache<span class="token punctuation">(</span><span class="token punctuation">)</span>
res4<span class="token operator">:</span> listRDD<span class="token punctuation">.</span><span class="token keyword">type</span> <span class="token operator">=</span> ParallelCollectionRDD<span class="token punctuation">[</span><span class="token number">0</span><span class="token punctuation">]</span> at parallelize at <span class="token operator">&lt;</span>console<span class="token operator">></span><span class="token operator">:</span><span class="token number">27</span>

scala<span class="token operator">></span> println<span class="token punctuation">(</span>listRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>mkString<span class="token punctuation">(</span><span class="token string">","</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
hadoop<span class="token punctuation">,</span>spark<span class="token punctuation">,</span>hive
</code></pre>
<h3 id="容错机制"><a href="#容错机制" class="headerlink" title="容错机制"></a>容错机制</h3><ol>
<li>血统方式：<ul>
<li>根据RDD之间的依赖关系</li>
</ul>
</li>
</ol>
<pre class=" language-a"><code class="language-a">RDD的原理
什么是RDD？
RDD如何来的？
RDD有什么用？
RDD是如何进行持久化的？
</code></pre>
<h2 id="Spark-SQL"><a href="#Spark-SQL" class="headerlink" title="Spark SQL"></a>Spark SQL</h2><pre class=" language-scala"><code class="language-scala"><span class="token comment" spellcheck="true">// DataFrame的创建</span>

<span class="token comment" spellcheck="true">// 通过toDF()创建</span>
<span class="token keyword">val</span> personRDD <span class="token operator">=</span> sc<span class="token punctuation">.</span>textFile<span class="token punctuation">(</span><span class="token string">"hdfs://node:9000/spark/test/person.txt"</span><span class="token punctuation">)</span>
<span class="token keyword">val</span> personDF <span class="token operator">=</span> personRDD<span class="token punctuation">.</span>toDF<span class="token punctuation">(</span><span class="token punctuation">)</span>

<span class="token keyword">val</span> lineRDD <span class="token operator">=</span> sc<span class="token punctuation">.</span>textFile<span class="token punctuation">(</span><span class="token string">"/spark/test/person.txt"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>map<span class="token punctuation">(</span>_<span class="token punctuation">.</span>split<span class="token punctuation">(</span><span class="token string">" "</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
<span class="token keyword">case</span> <span class="token keyword">class</span> Person<span class="token punctuation">(</span>id<span class="token operator">:</span><span class="token builtin">Int</span><span class="token punctuation">,</span>name<span class="token operator">:</span><span class="token builtin">String</span><span class="token punctuation">,</span>age<span class="token operator">:</span><span class="token builtin">Int</span><span class="token punctuation">)</span>
<span class="token keyword">val</span> personRDD <span class="token operator">=</span> lineRDD<span class="token punctuation">.</span>map<span class="token punctuation">(</span>x<span class="token keyword">=></span>Person<span class="token punctuation">(</span>x<span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">.</span>toInt<span class="token punctuation">,</span>x<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">,</span>x<span class="token punctuation">(</span><span class="token number">2</span><span class="token punctuation">)</span><span class="token punctuation">.</span>toInt<span class="token punctuation">)</span><span class="token punctuation">)</span>
<span class="token keyword">val</span> personDF <span class="token operator">=</span> personRDD<span class="token punctuation">.</span>toDF<span class="token punctuation">(</span><span class="token punctuation">)</span>

personDF<span class="token punctuation">.</span>select<span class="token punctuation">(</span><span class="token string">"name"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>show

<span class="token comment" spellcheck="true">// 2.直接创建</span>
<span class="token keyword">val</span> psersonDF2 <span class="token operator">=</span> spark<span class="token punctuation">.</span>read<span class="token punctuation">.</span>text<span class="token punctuation">(</span><span class="token string">"hdfs://node:9000/spark/test/person.txt"</span><span class="token punctuation">)</span>
</code></pre>
<p><img src="C:\Users\14533\AppData\Roaming\Typora\typora-user-images\image-20210507194417398.png" alt="image-20210507194417398"></p>
<p><img src="C:\Users\14533\AppData\Roaming\Typora\typora-user-images\image-20210507194732688.png" alt="image-20210507194732688"></p>
<p><img src="C:\Users\14533\AppData\Roaming\Typora\typora-user-images\image-20210507195029036.png" alt="image-20210507195029036"></p>
<p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210507195218684.png" alt="image-20210507195218684"></p>
<p><img src="C:\Users\14533\AppData\Roaming\Typora\typora-user-images\image-20210507200440435.png" alt="image-20210507200440435"></p>
<p><img src="C:\Users\14533\AppData\Roaming\Typora\typora-user-images\image-20210507200537271.png" alt="image-20210507200537271"></p>
<p><img src="C:\Users\14533\AppData\Roaming\Typora\typora-user-images\image-20210507201806664.png" alt="image-20210507201806664"></p>
<p><img src="C:\Users\14533\AppData\Roaming\Typora\typora-user-images\image-20210507202142525.png" alt="image-20210507202142525"></p>
<pre class=" language-scala"><code class="language-scala"><span class="token comment" spellcheck="true">// 1.将DataFrame注册成一个临时表</span>
personDF<span class="token punctuation">.</span>registerTempTable<span class="token punctuation">(</span><span class="token string">"t_person"</span><span class="token punctuation">)</span>

<span class="token comment" spellcheck="true">// 2.查询年龄最大的前两名人的信息</span>
spark<span class="token punctuation">.</span>sql<span class="token punctuation">(</span><span class="token string">"select * from t_person order by age desc limit 2"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>show<span class="token punctuation">(</span><span class="token punctuation">)</span>
</code></pre>
<h1 id="Spark学习笔记"><a href="#Spark学习笔记" class="headerlink" title="Spark学习笔记"></a>Spark学习笔记</h1><h2 id="第一章-Spark概述"><a href="#第一章-Spark概述" class="headerlink" title="第一章 Spark概述"></a>第一章 Spark概述</h2><h3 id="1-1-Spark是什么"><a href="#1-1-Spark是什么" class="headerlink" title="1.1 Spark是什么"></a>1.1 Spark是什么</h3><blockquote>
<p>Spark 是一种基于内存的快速、通用、可扩展的大数据分析计算引擎</p>
</blockquote>
<h3 id="1-2-Spark-or-Hadoop"><a href="#1-2-Spark-or-Hadoop" class="headerlink" title="1.2 Spark or Hadoop"></a>1.2 Spark or Hadoop</h3><ul>
<li>Hadoop MapReduce 由于其设计初衷并不是为了满足循环迭代式数据流处理，因此在多并行运行的数据可复用场景（如：机器学习、图挖掘算法、交互式数据挖掘算法）中存在诸多计算效率等问题。所以 Spark 应运而生，Spark 就是在传统的 MapReduce 计算框架的基础上，利用其计算过程的优化，从而大大加快了数据分析、挖掘的运行和读写速度，并将计算单元缩小到更适合并行计算和重复使用的 RDD 计算模型</li>
<li>机器学习中 ALS、凸优化梯度下降等。这些都需要基于数据集或者数据集的衍生数据反复查询反复操作。MR 这种模式不太合适，即使多 MR 串行处理，性能和时间也是一个问题。数据的共享依赖于磁盘。另外一种是交互式数据挖掘，MR 显然不擅长。而Spark 所基于的 scala 语言恰恰擅长函数的处理</li>
<li>Spark 是一个分布式数据快速分析项目。它的核心技术是弹性分布式数据集（Resilient Distributed Datasets），提供了比 MapReduce 丰富的模型，可以快速在内存中对数据集进行多次迭代，来支持复杂的数据挖掘算法和图形计算算法</li>
<li>==Spark 和Hadoop 的根本差异是多个作业之间的数据通信问题 : Spark 多个作业之间数据通信是基于内存，而 Hadoop 是基于磁盘==</li>
<li>Spark Task 的启动时间快。Spark 采用 fork 线程的方式，而 Hadoop 采用创建新的进程的方式。</li>
<li>Spark 只有在 shuffle 的时候将数据写入磁盘，而 Hadoop 中多个 MR 作业之间的数据交互都要依赖于磁盘交互</li>
<li>Spark 的缓存机制比 HDFS 的缓存机制高效</li>
</ul>
<h3 id="1-3-Spark核心模块"><a href="#1-3-Spark核心模块" class="headerlink" title="1.3 Spark核心模块"></a>1.3 Spark核心模块</h3><p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210504180944067.png" alt="Spark模块"></p>
<ul>
<li><p>Spark Core</p>
<blockquote>
<p>Spark Core中提供了Spark最基础与最核心的功能，Spark其他的功能如：Spark SQL。Spark Streaming，GraphX, MLlib都是在Spark Core的基础上进行扩展的</p>
</blockquote>
</li>
<li><p>Spark SQL</p>
<blockquote>
<p>Spark SQL是Spark用来操作结构化数据S的组件。通过Spark SQL，用户可以使用SQL或者Apache Hive版本的SQL方言（HQL）来查询数据</p>
</blockquote>
</li>
<li><p>Spark Streaming</p>
<blockquote>
<p>Spark Streaming是Spark平台上针对实时数据进行流式计算的组件，提供了丰富的处理数据流的API。</p>
</blockquote>
</li>
<li><p>Spark MLlib</p>
<blockquote>
<p>MLlib是Spark提供的一个机器学习算法库。MLlib不仅提供了模型评估、数据导入等额外的功能，还提供了一些更底层的机器学习原语。</p>
</blockquote>
</li>
<li><p>Spark GraphX</p>
<blockquote>
<p>GraphX是Spark面向图计算提供的框架与算法库。</p>
</blockquote>
</li>
</ul>
<h2 id="第二章-Spark快速入手"><a href="#第二章-Spark快速入手" class="headerlink" title="第二章 Spark快速入手"></a>第二章 Spark快速入手</h2><p><a target="_blank" rel="noopener" href="https://mvnrepository.com/">POM依赖下载地址</a></p>
<h3 id="2-1-创建Manven项目"><a href="#2-1-创建Manven项目" class="headerlink" title="2.1 创建Manven项目"></a>2.1 创建Manven项目</h3><h4 id="2-1-1-下载Scala插件"><a href="#2-1-1-下载Scala插件" class="headerlink" title="2.1.1 下载Scala插件"></a>2.1.1 下载Scala插件</h4><h4 id="2-1-2-依赖"><a href="#2-1-2-依赖" class="headerlink" title="2.1.2 依赖"></a>2.1.2 依赖</h4><pre class=" language-xml"><code class="language-xml"><span class="token prolog">&lt;?xml version="1.0" encoding="UTF-8"?></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>project</span> <span class="token attr-name">xmlns</span><span class="token attr-value"><span class="token punctuation">=</span><span class="token punctuation">"</span>http://maven.apache.org/POM/4.0.0<span class="token punctuation">"</span></span>
         <span class="token attr-name"><span class="token namespace">xmlns:</span>xsi</span><span class="token attr-value"><span class="token punctuation">=</span><span class="token punctuation">"</span>http://www.w3.org/2001/XMLSchema-instance<span class="token punctuation">"</span></span>
         <span class="token attr-name"><span class="token namespace">xsi:</span>schemaLocation</span><span class="token attr-value"><span class="token punctuation">=</span><span class="token punctuation">"</span>http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd<span class="token punctuation">"</span></span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>modelVersion</span><span class="token punctuation">></span></span>4.0.0<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>modelVersion</span><span class="token punctuation">></span></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.example<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>Spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>1.0-SNAPSHOT<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>properties</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>maven.compiler.source</span><span class="token punctuation">></span></span>8<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>maven.compiler.source</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>maven.compiler.target</span><span class="token punctuation">></span></span>8<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>maven.compiler.target</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>properties</span><span class="token punctuation">></span></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependencies</span><span class="token punctuation">></span></span>
        <span class="token comment" spellcheck="true">&lt;!--spark-core--></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-core_2.11<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>2.3.2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token comment" spellcheck="true">&lt;!--spark-streaming--></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-streaming_2.11<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>2.3.2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token comment" spellcheck="true">&lt;!-- spark-sql --></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-sql_2.11<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>2.3.2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token comment" spellcheck="true">&lt;!-- scala-libary 2.11.12 --></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.scala-lang<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>scala-library<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>2.11.8<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>


        <span class="token comment" spellcheck="true">&lt;!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka-0-10 --></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-streaming-kafka-0-10_2.11<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>2.4.4<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token comment" spellcheck="true">&lt;!-- spark-streaming-kafka --></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-streaming-kafka_2.11<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>1.6.3<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token comment" spellcheck="true">&lt;!-- spark-hive_2.11 --></span>
        <span class="token comment" spellcheck="true">&lt;!-- https://mvnrepository.com/artifact/org.apache.spark/spark-hive --></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-hive_2.11<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>2.3.2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token comment" spellcheck="true">&lt;!-- https://mvnrepository.com/artifact/org.apache.spark/spark-graphx --></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-graphx_2.11<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>2.3.2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.spark<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>spark-mllib_2.11<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>2.3.2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>scope</span><span class="token punctuation">></span></span>runtime<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>scope</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>
        <span class="token comment" spellcheck="true">&lt;!--        &lt;dependency>--></span>
        <span class="token comment" spellcheck="true">&lt;!--            &lt;groupId>org.apache.spark&lt;/groupId>--></span>
        <span class="token comment" spellcheck="true">&lt;!--            &lt;artifactId>spark-mllib_2.11&lt;/artifactId>--></span>
        <span class="token comment" spellcheck="true">&lt;!--            &lt;version>2.4.4&lt;/version>--></span>
        <span class="token comment" spellcheck="true">&lt;!--            &lt;scope>runtime&lt;/scope>--></span>
        <span class="token comment" spellcheck="true">&lt;!--        &lt;/dependency>--></span>

        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>mysql<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>mysql-connector-java<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>5.1.27<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span><span class="token comment" spellcheck="true">&lt;!--8.0.22--></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>log4j<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>log4j<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>1.2.17<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.slf4j<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>slf4j-api<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>1.7.21<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token comment" spellcheck="true">&lt;!--        &lt;dependency>--></span>
        <span class="token comment" spellcheck="true">&lt;!--            &lt;groupId>org.json4s&lt;/groupId>--></span>
        <span class="token comment" spellcheck="true">&lt;!--            &lt;artifactId>json4s-jackson_2.11&lt;/artifactId>--></span>
        <span class="token comment" spellcheck="true">&lt;!--            &lt;version>${json4s.version}&lt;/version>--></span>
        <span class="token comment" spellcheck="true">&lt;!--        &lt;/dependency>--></span>

        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>jfree<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>jfreechart<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>1.0.13<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>dependency</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.hadoop<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>hadoop-client<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>2.7.2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependency</span><span class="token punctuation">></span></span>

        <span class="token comment" spellcheck="true">&lt;!--        &lt;dependency>--></span>
        <span class="token comment" spellcheck="true">&lt;!--            &lt;groupId>org.apache.maven.plugins&lt;/groupId>--></span>
        <span class="token comment" spellcheck="true">&lt;!--            &lt;artifactId>maven-shade-plugin&lt;/artifactId>--></span>
        <span class="token comment" spellcheck="true">&lt;!--            &lt;version>2.3&lt;/version>--></span>
        <span class="token comment" spellcheck="true">&lt;!--        &lt;/dependency>--></span>

    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>dependencies</span><span class="token punctuation">></span></span>


    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>build</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>sourceDirectory</span><span class="token punctuation">></span></span>src/main/scala<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>sourceDirectory</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>testSourceDirectory</span><span class="token punctuation">></span></span>src/test/scala<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>testSourceDirectory</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>plugins</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>plugin</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>net.alchim31.maven<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>scala-maven-plugin<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>3.2.2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>executions</span><span class="token punctuation">></span></span>
                    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>execution</span><span class="token punctuation">></span></span>
                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>goals</span><span class="token punctuation">></span></span>
                            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>goal</span><span class="token punctuation">></span></span>compile<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>goal</span><span class="token punctuation">></span></span>
                            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>goal</span><span class="token punctuation">></span></span>testCompile<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>goal</span><span class="token punctuation">></span></span>
                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>goals</span><span class="token punctuation">></span></span>
                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
                            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>args</span><span class="token punctuation">></span></span>
                                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>arg</span><span class="token punctuation">></span></span>-dependencyfile<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>arg</span><span class="token punctuation">></span></span>
                                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>arg</span><span class="token punctuation">></span></span>${project.build.directory}/.scala_dependencies<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>arg</span><span class="token punctuation">></span></span>
                            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>args</span><span class="token punctuation">></span></span>
                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>
                    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>execution</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>executions</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>plugin</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>plugin</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>groupId</span><span class="token punctuation">></span></span>org.apache.maven.plugins<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>groupId</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifactId</span><span class="token punctuation">></span></span>maven-shade-plugin<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifactId</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>version</span><span class="token punctuation">></span></span>2.4.3<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>version</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>executions</span><span class="token punctuation">></span></span>
                    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>execution</span><span class="token punctuation">></span></span>
                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>phase</span><span class="token punctuation">></span></span>package<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>phase</span><span class="token punctuation">></span></span>
                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>goals</span><span class="token punctuation">></span></span>
                            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>goal</span><span class="token punctuation">></span></span>shade<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>goal</span><span class="token punctuation">></span></span>
                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>goals</span><span class="token punctuation">></span></span>
                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
                            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>filters</span><span class="token punctuation">></span></span>
                                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>filter</span><span class="token punctuation">></span></span>
                                    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>artifact</span><span class="token punctuation">></span></span>*:*<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>artifact</span><span class="token punctuation">></span></span>
                                    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>excludes</span><span class="token punctuation">></span></span>
                                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>exclude</span><span class="token punctuation">></span></span>META-INF/*.SF<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>exclude</span><span class="token punctuation">></span></span>
                                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>exclude</span><span class="token punctuation">></span></span>META-INF/*.DSA<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>exclude</span><span class="token punctuation">></span></span>
                                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>exclude</span><span class="token punctuation">></span></span>META-INF/*.RSA<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>exclude</span><span class="token punctuation">></span></span>
                                    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>excludes</span><span class="token punctuation">></span></span>
                                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>filter</span><span class="token punctuation">></span></span>
                            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>filters</span><span class="token punctuation">></span></span>
                            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>transformers</span><span class="token punctuation">></span></span>
                                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>transformer</span> <span class="token attr-name">implementation</span><span class="token attr-value"><span class="token punctuation">=</span><span class="token punctuation">"</span>org.apache.maven.plugins.shade.resource.ManifestResourceTransformer<span class="token punctuation">"</span></span><span class="token punctuation">></span></span>
                                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>transformer</span><span class="token punctuation">></span></span>
                            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>transformers</span><span class="token punctuation">></span></span>
                        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>
                    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>execution</span><span class="token punctuation">></span></span>
                <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>executions</span><span class="token punctuation">></span></span>
            <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>plugin</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>plugins</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>build</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>project</span><span class="token punctuation">></span></span>
</code></pre>
<h4 id="2-1-3-WordCount"><a href="#2-1-3-WordCount" class="headerlink" title="2.1.3 WordCount"></a>2.1.3 WordCount</h4><pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>wc

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>log4j<span class="token punctuation">.</span><span class="token punctuation">{</span>Level<span class="token punctuation">,</span> Logger<span class="token punctuation">}</span>
<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>RDD
<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token keyword">object</span> Spark03_WordCount <span class="token punctuation">{</span>

  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token comment" spellcheck="true">// Application</span>
    <span class="token comment" spellcheck="true">// Spark框架</span>
    <span class="token comment" spellcheck="true">// TODO 建立和Spark框架的连接</span>
    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"WordCount"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO　执行业务操作</span>
    <span class="token keyword">val</span> lines<span class="token operator">:</span> RDD<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span> <span class="token operator">=</span> sc<span class="token punctuation">.</span>textFile<span class="token punctuation">(</span><span class="token string">"datas"</span><span class="token punctuation">)</span>

    <span class="token keyword">val</span> words <span class="token operator">=</span> lines<span class="token punctuation">.</span>flatMap<span class="token punctuation">(</span>_<span class="token punctuation">.</span>split<span class="token punctuation">(</span><span class="token string">" "</span><span class="token punctuation">)</span><span class="token punctuation">)</span>

    <span class="token keyword">val</span> wordToOne <span class="token operator">=</span> words<span class="token punctuation">.</span>map<span class="token punctuation">(</span>
      word <span class="token keyword">=></span> <span class="token punctuation">(</span>word<span class="token punctuation">,</span> <span class="token number">1</span><span class="token punctuation">)</span>
    <span class="token punctuation">)</span>
    <span class="token comment" spellcheck="true">// Spark框架提供了更多的功能，可以将分组和聚合使用一个方法实现</span>
    <span class="token comment" spellcheck="true">// reduceByKey()：相同的key的数据，可以对value进行reduce聚合</span>
    <span class="token comment" spellcheck="true">// wordToOne.reduceByKey((x, y) => {x + y})</span>
    <span class="token keyword">val</span> wordToCount <span class="token operator">=</span> wordToOne<span class="token punctuation">.</span>reduceByKey<span class="token punctuation">(</span>_<span class="token operator">+</span>_<span class="token punctuation">)</span>

    <span class="token keyword">val</span> array <span class="token operator">=</span> wordToCount<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span>
    array<span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 关闭连接</span>
    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<h4 id="2-1-4-log4j-properties-配置日志文件"><a href="#2-1-4-log4j-properties-配置日志文件" class="headerlink" title="2.1.4 log4j.properties 配置日志文件"></a>2.1.4 log4j.properties 配置日志文件</h4><pre class=" language-properties"><code class="language-properties"><span class="token attr-name">log4j.rootCategory</span><span class="token punctuation">=</span><span class="token attr-value">ERROR, console</span>
<span class="token attr-name">log4j.appender.console</span><span class="token punctuation">=</span><span class="token attr-value">org.apache.log4j.ConsoleAppender</span>
<span class="token attr-name">log4j.appender.console.target</span><span class="token punctuation">=</span><span class="token attr-value">System.err</span>
<span class="token attr-name">log4j.appender.console.layout</span><span class="token punctuation">=</span><span class="token attr-value">org.apache.log4j.PatternLayout</span>
<span class="token attr-name">log4j.appender.console.layout.ConversionPattern</span><span class="token punctuation">=</span><span class="token attr-value">%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n</span>

<span class="token comment" spellcheck="true"># Set the default spark-shell log level to ERROR. When running the spark-shell,the</span>
<span class="token comment" spellcheck="true"># log level for this class is used to overwrite the root logger's log level, so that</span>
<span class="token comment" spellcheck="true"># the user can have different defaults for the shell and regular Spark apps.</span>
<span class="token attr-name">log4j.logger.org.apache.spark.repl.Main</span><span class="token punctuation">=</span><span class="token attr-value">ERROR</span>

<span class="token comment" spellcheck="true"># Settings to quiet third party logs that are too verbose</span>
<span class="token attr-name">log4j.logger.org.spark_project.jetty</span><span class="token punctuation">=</span><span class="token attr-value">ERROR</span>
<span class="token attr-name">log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle</span><span class="token punctuation">=</span><span class="token attr-value">ERROR</span>
<span class="token attr-name">log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper</span><span class="token punctuation">=</span><span class="token attr-value">ERROR</span>
<span class="token attr-name">log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter</span><span class="token punctuation">=</span><span class="token attr-value">ERROR</span>
<span class="token attr-name">log4j.logger.org.apache.parquet</span><span class="token punctuation">=</span><span class="token attr-value">ERROR</span>
<span class="token attr-name">log4j.logger.parquet</span><span class="token punctuation">=</span><span class="token attr-value">ERROR</span>
</code></pre>
<h2 id="第三章-Spark运行环境"><a href="#第三章-Spark运行环境" class="headerlink" title="第三章 Spark运行环境"></a>第三章 Spark运行环境</h2><blockquote>
<p>Spark 作为一个数据处理框架和计算引擎，被设计在所有常见的集群环境中运行, 在国内工作中主流的环境为 Yarn，不过逐渐容器式环境也慢慢流行起来。接下来，我们就分别看看不同环境下 Spark 的运行</p>
</blockquote>
<p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210504190306359.png" alt="运行环境"></p>
<pre class=" language-scala"><code class="language-scala">bin<span class="token operator">/</span>spark<span class="token operator">-</span>submit <span class="token operator">--</span><span class="token keyword">class</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>examples<span class="token punctuation">.</span>SparkPi <span class="token operator">--</span>master spark<span class="token operator">:</span><span class="token operator">/</span><span class="token operator">/</span>node<span class="token operator">:</span><span class="token number">7077</span> examples<span class="token operator">/</span>jars<span class="token operator">/</span>spark<span class="token operator">-</span>examples_2<span class="token number">.11</span><span class="token operator">-</span><span class="token number">2.3</span><span class="token punctuation">.</span><span class="token number">2</span><span class="token punctuation">.</span>jar <span class="token number">10</span>


bin<span class="token operator">/</span>spark<span class="token operator">-</span>submit <span class="token operator">--</span><span class="token keyword">class</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>examples<span class="token punctuation">.</span>SparkPi <span class="token operator">--</span>master yarn <span class="token operator">--</span>deploy<span class="token operator">-</span>mode cluster examples<span class="token operator">/</span>jars<span class="token operator">/</span>spark<span class="token operator">-</span>examples_2<span class="token number">.11</span><span class="token operator">-</span><span class="token number">2.3</span><span class="token punctuation">.</span><span class="token number">2</span><span class="token punctuation">.</span>jar <span class="token number">10</span>


bin<span class="token operator">/</span>spark<span class="token operator">-</span>submit <span class="token operator">--</span><span class="token keyword">class</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>examples<span class="token punctuation">.</span>SparkPi <span class="token operator">--</span>master yarn <span class="token operator">--</span>deploy<span class="token operator">-</span>mode client examples<span class="token operator">/</span>jars<span class="token operator">/</span>spark<span class="token operator">-</span>examples_2<span class="token number">.11</span><span class="token operator">-</span><span class="token number">2.3</span><span class="token punctuation">.</span><span class="token number">2</span><span class="token punctuation">.</span>jar <span class="token number">10</span>
</code></pre>
<h2 id="第四章-Spark运行架构"><a href="#第四章-Spark运行架构" class="headerlink" title="第四章 Spark运行架构"></a>第四章 Spark运行架构</h2><h3 id="4-1-运行架构"><a href="#4-1-运行架构" class="headerlink" title="4.1 运行架构"></a>4.1 运行架构</h3><blockquote>
<p>Spark 框架的核心是一个计算引擎，整体来说，它采用了标准 master-slave 的结构</p>
</blockquote>
<ul>
<li>图形中的 Driver 表示 master，负责管理整个集群中的作业任务调度。图形中的 Executor 则是 slave，负责实际执行任务</li>
</ul>
<p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210505115734291.png" alt="运行架构"></p>
<h3 id="4-2-核心组件"><a href="#4-2-核心组件" class="headerlink" title="4.2 核心组件"></a>4.2 核心组件</h3><h4 id="4-2-1-Driver"><a href="#4-2-1-Driver" class="headerlink" title="4.2.1 Driver"></a>4.2.1 Driver</h4><p>Spark驱动器节点，用于执行Spark任务中的main方法，负责实际代码的执行工作。Driver在Spark作业执行时主要负责：</p>
<ol>
<li>将用户程序转化为作业（job）</li>
<li>在Executor之间调度任务(task)</li>
<li>跟踪Executor的执行情况</li>
<li>通过UI展示查询运行情况</li>
</ol>
<h4 id="4-2-2-Executor"><a href="#4-2-2-Executor" class="headerlink" title="4.2.2 Executor"></a>4.2.2 Executor</h4><blockquote>
<p>Spark Executor是集群中工作节点（Worker）中的一个JVM进程，负责在Spark 作业中运行具体任务（Task），任务彼此之间相互独立。Spark 应用启动时，Executor节点被同时启动，并且始终伴随着整个Spark 应用的生命周期而存在。如果有Executor节点发生了故障或崩溃，Spark 应用也可以继续执行，会将出错节点上的任务调度到其他Executor节点上继续运行。</p>
</blockquote>
<ul>
<li>Executor有两个核心功能：<ul>
<li>负责运行组成Spark应用的任务，并将结果返回给驱动器进程</li>
<li>它们通过自身的块管理器（Block Manager）为用户程序中要求缓存的RDD 提供内存式存储。RDD 是直接缓存在Executor进程内的，因此任务可以在运行时充分利用缓存数据加速运算。</li>
</ul>
</li>
</ul>
<h4 id="4-2-3-Master-amp-Worker"><a href="#4-2-3-Master-amp-Worker" class="headerlink" title="4.2.3 Master &amp; Worker"></a>4.2.3 Master &amp; Worker</h4><blockquote>
<p>Spark集群的独立部署环境中，不需要依赖其他的资源调度框架，自身就实现了资源调度的功能，所以环境中还有其他两个核心组件：Master和Worker，这里的Master是一个进程，主要负责资源的调度和分配，并进行集群的监控等职责，类似于Yarn环境中的RM, 而Worker呢，也是进程，一个Worker运行在集群中的一台服务器上，由Master分配资源对数据进行并行的处理和计算，类似于Yarn环境中NM。</p>
</blockquote>
<h4 id="4-2-4-ApplicationMaster"><a href="#4-2-4-ApplicationMaster" class="headerlink" title="4.2.4 ApplicationMaster"></a>4.2.4 ApplicationMaster</h4><blockquote>
<p>Hadoop用户向YARN集群提交应用程序时,提交程序中应该包含ApplicationMaster，用于向资源调度器申请执行任务的资源容器Container，运行用户自己的程序任务job，监控整个任务的执行，跟踪整个任务的状态，处理任务失败等异常情况</p>
</blockquote>
<h3 id="4-3-核心概念"><a href="#4-3-核心概念" class="headerlink" title="4.3 核心概念"></a>4.3 核心概念</h3><h4 id="4-3-1-Executor-与-Core"><a href="#4-3-1-Executor-与-Core" class="headerlink" title="4.3.1 Executor 与 Core"></a>4.3.1 Executor 与 Core</h4><blockquote>
<p>Spark Executor是集群中运行在工作节点（Worker）中的一个JVM进程，是整个集群中的专门用于计算的节点。在提交应用中，可以提供参数指定计算节点的个数，以及对应的资源。这里的资源一般指的是工作节点Executor的内存大小和使用的虚拟CPU核（Core）数量。</p>
</blockquote>
<p>应用程序相关启动参数如下：</p>
<table>
<thead>
<tr>
<th>名称</th>
<th>说明</th>
</tr>
</thead>
<tbody><tr>
<td>–num-executors</td>
<td>配置Executor的数量</td>
</tr>
<tr>
<td>–executor-memory</td>
<td>配置每个Executor的内存大小</td>
</tr>
<tr>
<td>–executor-cores</td>
<td>配置每个Executor的虚拟CPU core数量</td>
</tr>
</tbody></table>
<h4 id="4-3-2-并行度（Parallelism）"><a href="#4-3-2-并行度（Parallelism）" class="headerlink" title="4.3.2 并行度（Parallelism）"></a>4.3.2 并行度（Parallelism）</h4><blockquote>
<p>在分布式计算框架中一般都是多个任务同时执行，由于任务分布在不同的计算节点进行计算，所以能够真正地实现多任务并行执行，记住，这里是并行，而不是并发。这里我们将整个集群并行执行任务的数量称之为<code>并行度</code>。那么一个作业到底并行度是多少呢？这个取决于框架的默认配置。应用程序也可以在运行过程中动态修改。</p>
</blockquote>
<h4 id="4-3-3-有向无环图（DAG）"><a href="#4-3-3-有向无环图（DAG）" class="headerlink" title="4.3.3 有向无环图（DAG）"></a>4.3.3 有向无环图（DAG）</h4><p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210505120842451.png" alt="有向无环图"></p>
<h3 id="4-4-提交流程"><a href="#4-4-提交流程" class="headerlink" title="4.4 提交流程"></a>4.4 提交流程</h3><blockquote>
<p>提交流程，其实就是我们开发人员根据需求写的应用程序通过Spark客户端提交给Spark运行环境执行计算的流程。</p>
</blockquote>
<p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210505121201263.png" alt="提交流程图"></p>
<p>Spark应用程序提交到Yarn环境中执行的时候，一般会有两种部署执行的方式：Client 和Cluster。两种模式主要区别在于：Driver程序的运行节点位置。</p>
<h4 id="4-2-1-Yarn-Client-模式"><a href="#4-2-1-Yarn-Client-模式" class="headerlink" title="4.2.1 Yarn Client 模式"></a>4.2.1 Yarn Client 模式</h4><blockquote>
<p>Client模式将用于监控和调度的Driver模块在客户端执行，而不是在Yarn中，所以一般用于测试</p>
</blockquote>
<ul>
<li><p><code>Driver</code>在任务提交的本地机器上运行</p>
</li>
<li><p><code>Driver</code>启动后会和<code>ResourceManager</code>通讯申请启动<code>ApplicationMaster</code></p>
</li>
<li><p><code>ResourceManager</code>分配<code>container</code>，在合适的<code>NodeManager</code>上启动<code>ApplicationMaster</code>，负责向<code>ResourceManager</code>申请<code>Executor</code>内存</p>
</li>
<li><p><code>ResourceManager</code>接到<code>ApplicationMaster</code>的资源申请后会分配<code>container</code>，然后 <code>ApplicationMaster</code>在资源分配指定的<code>NodeManager</code>上启动<code>Executor</code>进程</p>
</li>
<li><p><code>Executor</code>进程启动后会向<code>Driver</code>反向注册，<code>Executor</code>全部注册完成后<code>Driver</code>开始执行<code>main</code>函数</p>
</li>
<li><p>之后执行到<code>Action</code>算子时，触发一个<code>Job</code>，并根据宽依赖开始划分<code>stage</code>，每个<code>stage</code>生成对应的<code>TaskSet</code>，之后将<code>task</code>分发到各个<code>Executor</code>上执行</p>
</li>
</ul>
<h4 id="4-2-2-Yarn-Cluster-模式"><a href="#4-2-2-Yarn-Cluster-模式" class="headerlink" title="4.2.2 Yarn Cluster 模式"></a>4.2.2 Yarn Cluster 模式</h4><blockquote>
<p>Cluster模式将用于监控和调度的Driver模块启动在Yarn集群资源中执行。一般应用于实际生产环境。</p>
</blockquote>
<ul>
<li>在<code>YARN Cluster</code>模式下，任务提交后会和<code>ResourceManager</code>通讯申请启动 <code>ApplicationMaster</code></li>
<li>随后<code>ResourceManager</code>分配<code>container</code>，在合适的<code>NodeManager</code>上启动<code>ApplicationMaster</code>，此时的<code>ApplicationMaster</code>就是<code>Driver</code></li>
<li><code>Driver</code>启动后向<code>ResourceManager</code>申请<code>Executor</code>内存，<code>ResourceManager</code>接到 <code>ApplicationMaster</code>的资源申请后会分配<code>container</code>，然后在合适的<code>NodeManager</code>上启动 <code>Executor</code>进程</li>
<li><code>Executor</code>进程启动后会向<code>Driver</code>反向注册，<code>Executor</code>全部注册完成后Driver开始执行<code>main</code>函数</li>
<li>之后执行到<code>Action</code>算子时，触发一个<code>Job</code>，并根据宽依赖开始划分<code>stage</code>，每个<code>stage</code>生成对应的<code>TaskSet</code>，之后将<code>task</code>分发到各个<code>Executor</code>上执行</li>
</ul>
<h2 id="第五章-Saprk核心编程"><a href="#第五章-Saprk核心编程" class="headerlink" title="第五章 Saprk核心编程"></a>第五章 Saprk核心编程</h2><p>Spark计算框架为了能够进行高并发和高吞吐的数据处理，封装了三大数据结构，用于处理不同的应用场景。三大数据结构分别是：</p>
<ul>
<li>==RDD==：弹性分布式数据集</li>
<li>==累加器==：分布式共享只写变量</li>
<li>==广播变量==：分布式共享只读变量</li>
</ul>
<h3 id="5-1-RDD"><a href="#5-1-RDD" class="headerlink" title="5.1 RDD"></a>5.1 RDD</h3><h4 id="5-1-1-什么是RDD"><a href="#5-1-1-什么是RDD" class="headerlink" title="5.1.1 什么是RDD"></a>5.1.1 什么是RDD</h4><blockquote>
<p>RDD（Resilient Distributed Dataset）叫做弹性分布式数据集，是 Spark 中最基本的数据处理模型。代码中是一个抽象类，它代表一个弹性的、不可变、可分区、里面的元素可并行计算的集合</p>
</blockquote>
<ul>
<li><p>弹性</p>
<ul>
<li>存储的弹性：内存与磁盘的自动切换</li>
<li>容错的弹性：数据丢失可以自动恢复</li>
<li>计算的弹性：计算出错重试机制</li>
<li>分片的弹性：可根据需要重新分片</li>
</ul>
</li>
<li><p>分布式：数据存储在大数据集群不同节点上</p>
</li>
<li><p>数据集：RDD 封装了计算逻辑，并不保存数据</p>
</li>
<li><p>数据抽象：RDD 是一个抽象类，需要子类具体实现</p>
</li>
<li><p>不可变：RDD 封装了计算逻辑，是不可以改变的，想要改变，只能产生新的 RDD，在新的 RDD 里面封装计算逻辑</p>
</li>
<li><p>可分区、并行计算</p>
</li>
</ul>
<h4 id="5-1-2-核心属性"><a href="#5-1-2-核心属性" class="headerlink" title="5.1.2 核心属性"></a>5.1.2 核心属性</h4><p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210505215427584.png" alt="核心属性"></p>
<ul>
<li>分区列表<ul>
<li>RDD数据结构中存在分区列表，用于执行任务时并行计算，是实现分布式计算的重要属性</li>
</ul>
</li>
</ul>
<p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210505215523996.png" alt="分区列表"></p>
<ul>
<li>分区计算函数<ul>
<li>Spark在计算时，是使用分区函数对每一个分区进行计算</li>
</ul>
</li>
</ul>
<p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210505215616152.png" alt="分区计算函数"></p>
<ul>
<li>RDD之间的依赖关系<ul>
<li>RDD是计算模型的封装，当需求中需要将多个计算模型进行组合时，就需要将多个RDD建立依赖关系</li>
</ul>
</li>
</ul>
<p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210505215655411.png" alt="依赖关系"></p>
<ul>
<li>分区器（可选）<ul>
<li>当数据为KV类型数据时，可以通过设定分区器自定义数据的分区</li>
</ul>
</li>
</ul>
<p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210505215728608.png" alt="分区器"></p>
<ul>
<li> 首选位置（可选）</li>
<li>计算数据时，可以根据计算节点的状态选择不同的节点位置进行计算</li>
</ul>
<p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/image-20210505215811647.png" alt="首选位置"></p>
<h4 id="5-1-3-执行原理"><a href="#5-1-3-执行原理" class="headerlink" title="5.1.3 执行原理"></a>5.1.3 执行原理</h4><blockquote>
<p>RDD在整个流程中主要用于将逻辑进行封装，并生成Task发送给Executor节点执行计算</p>
</blockquote>
<h3 id="5-2-基础编程"><a href="#5-2-基础编程" class="headerlink" title="5.2 基础编程"></a>5.2 基础编程</h3><h4 id="5-2-1-RDD创建"><a href="#5-2-1-RDD创建" class="headerlink" title="5.2.1 RDD创建"></a>5.2.1 RDD创建</h4><blockquote>
<p>在Spark中创建RDD的创建方式可以分为四种</p>
</blockquote>
<p>1）从集合(内存)中创建RDD</p>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>builder

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>RDD
<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token keyword">object</span> Spark01_RDD_Memory <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>
    <span class="token comment" spellcheck="true">// TODO 主备环境</span>
    <span class="token comment" spellcheck="true">// * 表示当前线程可用的最大核数</span>
    <span class="token keyword">val</span> conf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"RDD"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>conf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO　创建RDD</span>
    <span class="token comment" spellcheck="true">// 从内存中创建RDD，将文件中的数据作为处理的数据源</span>
    <span class="token keyword">val</span> seq <span class="token operator">=</span> Seq<span class="token punctuation">[</span><span class="token builtin">Int</span><span class="token punctuation">]</span><span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span> <span class="token number">2</span><span class="token punctuation">,</span> <span class="token number">3</span><span class="token punctuation">,</span> <span class="token number">4</span><span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// parallelize：并行</span>
    <span class="token comment" spellcheck="true">//val rdd: RDD[Int] = sc.parallelize(sep)</span>
    <span class="token comment" spellcheck="true">// makeRDD方法在底层实现时其实就是调用了rdd对象的parallelize方法</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>seq<span class="token punctuation">)</span>

    rdd<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>
<span class="token comment" spellcheck="true">//    println(rdd)</span>
    <span class="token comment" spellcheck="true">// TODO 关闭环境</span>
    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>

  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<p>2）从外部存储(文件)创建RDD</p>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>builder

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token keyword">object</span> Spark02_RDD_File <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>
    <span class="token comment" spellcheck="true">// TODO 主备环境</span>
    <span class="token comment" spellcheck="true">// * 表示当前线程可用的最大核数</span>
    <span class="token keyword">val</span> conf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"RDD"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>conf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO　创建RDD</span>
    <span class="token comment" spellcheck="true">/*
     从文件中创建RDD，将内存中集合的数据作为处理的数据源
     path路径默认以当前环境的根路径为基准，可以写绝对路径或相对路径
     path路径可以时文件的具体路径，也可以是目录名称
     path路径还可以使用通配符 *
     path还可以是分布式存储系统路径：HDFS  sc.textFile("hdfs://node:9000/data")
     */</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>textFile<span class="token punctuation">(</span><span class="token string">"datas"</span><span class="token punctuation">)</span>
    rdd<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>
    <span class="token comment" spellcheck="true">// TODO 关闭环境</span>
    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>

  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<p>3）从其他RDD创建</p>
<blockquote>
<p>主要是通过一个RDD运算完后，再产生新的RDD</p>
</blockquote>
<p>4）直接创建RDD（new）</p>
<blockquote>
<p>使用new的方式直接构造RDD，一般由Spark框架自身使用</p>
</blockquote>
<h4 id="5-2-2-RDD转换算子"><a href="#5-2-2-RDD转换算子" class="headerlink" title="5.2.2 RDD转换算子"></a>5.2.2 RDD转换算子</h4><blockquote>
<p>RDD 根据数据处理方式的不同将算子整体上分为 Value 类型、双 Value 类型和 Key-Value类型</p>
</blockquote>
<p><strong>value 类型</strong></p>
<h5 id="1-map"><a href="#1-map" class="headerlink" title="1.map"></a>1.map</h5><blockquote>
<p>将处理的数据逐条进行映射转换，这里的转换可以是类型的转换，也可以是值的转换</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token keyword">object</span> Spark01_RDD_Operator_map_Transform <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - map</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span> <span class="token number">2</span><span class="token punctuation">,</span> <span class="token number">3</span><span class="token punctuation">,</span> <span class="token number">4</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
    <span class="token comment" spellcheck="true">// 1,2,3,4 => 2,4,6,8</span>

<span class="token comment" spellcheck="true">//    def mapFunction(num: Int): Int = {</span>
<span class="token comment" spellcheck="true">//      num * 2</span>
<span class="token comment" spellcheck="true">//    }</span>
<span class="token comment" spellcheck="true">//    val mapRdd = rdd.map(mapFunction)</span>

<span class="token comment" spellcheck="true">//    val mapRdd = rdd.map((num: Int)=>{num*2})</span>
<span class="token comment" spellcheck="true">//    val mapRdd = rdd.map((num: Int)=>num*2)</span>
<span class="token comment" spellcheck="true">//    val mapRdd = rdd.map(num=>num*2)</span>
    <span class="token keyword">val</span> mapRdd <span class="token operator">=</span> rdd<span class="token punctuation">.</span>map<span class="token punctuation">(</span>_<span class="token operator">*</span><span class="token number">2</span><span class="token punctuation">)</span>
    mapRdd<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>
    
    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>
<span class="token comment" spellcheck="true">/*
从服务器日志数据 apache.log 中获取用户请求 URL 资源路径
 */</span>
<span class="token keyword">object</span> Spark01_RDD_Operator_map_Transform_Test <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - map</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>textFile<span class="token punctuation">(</span><span class="token string">"datas/apache.log"</span><span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// 长的字符串转为短的字符串</span>
<span class="token comment" spellcheck="true">//    val mapRDD = rdd.map(</span>
<span class="token comment" spellcheck="true">//      line => {</span>
<span class="token comment" spellcheck="true">//        val datas = line.split(" ")</span>
<span class="token comment" spellcheck="true">//        datas(6)</span>
<span class="token comment" spellcheck="true">//      }</span>
<span class="token comment" spellcheck="true">//    )</span>

<span class="token comment" spellcheck="true">//    val mapRDD = rdd.map(line=>line.split(" ")(6))</span>
    <span class="token keyword">val</span> mapRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>map<span class="token punctuation">(</span>_<span class="token punctuation">.</span>split<span class="token punctuation">(</span><span class="token string">" "</span><span class="token punctuation">)</span><span class="token punctuation">(</span><span class="token number">6</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
    mapRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>

    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<h5 id="2-mapPartitions"><a href="#2-mapPartitions" class="headerlink" title="2.mapPartitions"></a>2.mapPartitions</h5><blockquote>
<p>将待处理的数据以分区为单位发送到计算节点进行处理，这里的处理是指可以进行任意的处理，哪怕是过滤数据。</p>
</blockquote>
<ul>
<li>map 和 mapPartitions 的区别？<ul>
<li>Map 算子是分区内一个数据一个数据的执行，类似于串行操作。而 mapPartitions 算子是以分区为单位进行批处理操作。</li>
<li>Map 算子主要目的将数据源中的数据进行转换和改变。但是不会减少或增多数据。MapPartitions 算子需要传递一个迭代器，返回一个迭代器，没有要求的元素的个数保持不变，所以可以增加或减少数据</li>
<li>Map 算子因为类似于串行操作，所以性能比较低，而是 mapPartitions 算子类似于批处理，所以性能较高。但是 mapPartitions 算子会长时间占用内存，那么这样会导致内存可能不够用，出现内存溢出的错误。所以在内存有限的情况下，不推荐使用。使用 map 操作。</li>
</ul>
</li>
</ul>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>
<span class="token comment" spellcheck="true">/*
  取出每个分区的最大值
 */</span>
<span class="token keyword">object</span> Spark02_RDD_Operator_mapPartitions_Transform_Test <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - mapPartitions</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span><span class="token punctuation">)</span><span class="token punctuation">,</span> <span class="token number">2</span><span class="token punctuation">)</span>


<span class="token comment" spellcheck="true">//    val mapRDD = rdd.mapPartitions(</span>
<span class="token comment" spellcheck="true">//      iter => {</span>
<span class="token comment" spellcheck="true">//        List(iter.max).iterator</span>
<span class="token comment" spellcheck="true">//      }</span>
<span class="token comment" spellcheck="true">//    )</span>

    <span class="token keyword">val</span> mapRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>mapPartitions<span class="token punctuation">(</span>iter<span class="token keyword">=></span>List<span class="token punctuation">(</span>iter<span class="token punctuation">.</span>max<span class="token punctuation">)</span><span class="token punctuation">.</span>iterator<span class="token punctuation">)</span>

    mapRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>

    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<h5 id="3-mapPertitionsWithIndex"><a href="#3-mapPertitionsWithIndex" class="headerlink" title="3.mapPertitionsWithIndex"></a>3.mapPertitionsWithIndex</h5><blockquote>
<p>将待处理的数据以分区为单位发送到计算节点进行处理，这里的处理是指可以进行任意的处理，哪怕是过滤数据，在处理时同时可以获取当前分区索引。</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>
<span class="token comment" spellcheck="true">/*
  取出指定分区的数据
 */</span>
<span class="token keyword">object</span> Spark03_RDD_Operator_mapPartitionsWithIndex_Transform <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - mapPartitionsWithIndex</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span><span class="token punctuation">)</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> mpiRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>mapPartitionsWithIndex<span class="token punctuation">(</span>
      <span class="token punctuation">(</span>index<span class="token punctuation">,</span> iter<span class="token punctuation">)</span> <span class="token keyword">=></span> <span class="token punctuation">{</span>
        <span class="token keyword">if</span> <span class="token punctuation">(</span>index <span class="token operator">==</span> <span class="token number">1</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
          iter
        <span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token punctuation">{</span>
          Nil<span class="token punctuation">.</span>iterator
        <span class="token punctuation">}</span>
      <span class="token punctuation">}</span>
    <span class="token punctuation">)</span>

    mpiRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>
    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  取出的数据以及对应的分区号
 */</span>
<span class="token keyword">object</span> Spark03_RDD_Operator_mapPartitionsWithIndex_Transform1 <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - mapPartitionsWithIndex</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> mpiRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>mapPartitionsWithIndex<span class="token punctuation">(</span>
      <span class="token punctuation">(</span>index<span class="token punctuation">,</span> iter<span class="token punctuation">)</span> <span class="token keyword">=></span> <span class="token punctuation">{</span>
        iter<span class="token punctuation">.</span>map<span class="token punctuation">(</span>
          num <span class="token keyword">=></span> <span class="token punctuation">{</span>
            <span class="token punctuation">(</span>num<span class="token punctuation">,</span> index<span class="token punctuation">)</span>
          <span class="token punctuation">}</span>
        <span class="token punctuation">)</span>
      <span class="token punctuation">}</span>
    <span class="token punctuation">)</span>

    mpiRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>
    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<h5 id="4-flatMap"><a href="#4-flatMap" class="headerlink" title="4.flatMap"></a>4.flatMap</h5><blockquote>
<p>将处理的数据进行扁平化后再进行映射处理，所以算子也称之为扁平映射</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  取出指定分区的数据
 */</span>
<span class="token keyword">object</span> Spark04_RDD_Operator_flatMap_Transform <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - flatMap</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>
      List<span class="token punctuation">(</span>
        List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">)</span><span class="token punctuation">,</span> List<span class="token punctuation">(</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span><span class="token punctuation">)</span>
      <span class="token punctuation">)</span>
    <span class="token punctuation">)</span>
    rdd<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>

<span class="token comment" spellcheck="true">//    val flatRDD = rdd.flatMap(</span>
<span class="token comment" spellcheck="true">//      list => {</span>
<span class="token comment" spellcheck="true">//        list</span>
<span class="token comment" spellcheck="true">//      }</span>
<span class="token comment" spellcheck="true">//    )</span>
    <span class="token keyword">val</span> flatRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>flatMap<span class="token punctuation">(</span>list<span class="token keyword">=></span>list<span class="token punctuation">)</span>

    flatRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>
    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  模式匹配
 */</span>
<span class="token keyword">object</span> Spark04_RDD_Operator_flatMap_Transform2 <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - flatMap</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">)</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span>List<span class="token punctuation">(</span><span class="token number">4</span><span class="token punctuation">,</span><span class="token number">5</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">)</span>

    <span class="token keyword">val</span> flatRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>flatMap <span class="token punctuation">{</span>
      <span class="token keyword">case</span> list<span class="token operator">:</span> List<span class="token punctuation">[</span>_<span class="token punctuation">]</span> <span class="token keyword">=></span> list
      <span class="token keyword">case</span> dat <span class="token keyword">=></span> List<span class="token punctuation">(</span>dat<span class="token punctuation">)</span>
    <span class="token punctuation">}</span>
    flatRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>

    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<h5 id="5-glom"><a href="#5-glom" class="headerlink" title="5.glom"></a>5.glom</h5><blockquote>
<p>将同一个分区的数据直接转换为相同类型的内存数组进行处理，分区不变</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>RDD
<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  glom
  小功能：计算所有分区最大值求和（分区内取最大值，分区间最大值求和）
 */</span>
<span class="token keyword">object</span> Spark05_RDD_Operator_glom_Transform_Test <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - glom</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span><span class="token punctuation">)</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">)</span>
    <span class="token comment" spellcheck="true">// List => Int</span>
    <span class="token comment" spellcheck="true">// Int => Array</span>
    <span class="token keyword">val</span> glomRDD<span class="token operator">:</span> RDD<span class="token punctuation">[</span>Array<span class="token punctuation">[</span><span class="token builtin">Int</span><span class="token punctuation">]</span><span class="token punctuation">]</span> <span class="token operator">=</span> rdd<span class="token punctuation">.</span>glom<span class="token punctuation">(</span><span class="token punctuation">)</span>

    <span class="token keyword">val</span> maxRDD <span class="token operator">=</span> glomRDD<span class="token punctuation">.</span>map<span class="token punctuation">(</span>_<span class="token punctuation">.</span>max<span class="token punctuation">)</span>
    maxRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span> <span class="token comment" spellcheck="true">// 分区最大值</span>
    println<span class="token punctuation">(</span>maxRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>sum<span class="token punctuation">)</span>  <span class="token comment" spellcheck="true">// 分区最大值求和</span>

    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<h5 id="6-groupBy"><a href="#6-groupBy" class="headerlink" title="6.groupBy"></a>6.groupBy</h5><blockquote>
<p>将数据根据指定的规则进行分组, 分区默认不变，但是数据会被打乱重新组合，我们将这样的操作称之为 shuffle。极限情况下，数据可能被分在同一个分区中</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  groupBy分组，按奇偶数划分
 */</span>
<span class="token keyword">object</span> Spark06_RDD_Operator_groupBy_Transform <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - groupBy</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span><span class="token punctuation">)</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">)</span>
    rdd<span class="token punctuation">.</span>saveAsTextFile<span class="token punctuation">(</span><span class="token string">"output"</span><span class="token punctuation">)</span>
    <span class="token keyword">def</span> groupFunction<span class="token punctuation">(</span>num<span class="token operator">:</span> <span class="token builtin">Int</span><span class="token punctuation">)</span> <span class="token operator">=</span> <span class="token punctuation">{</span>
      num <span class="token operator">%</span> <span class="token number">2</span>
    <span class="token punctuation">}</span>
    <span class="token keyword">val</span> groupRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>groupBy<span class="token punctuation">(</span>groupFunction<span class="token punctuation">)</span>

    <span class="token keyword">val</span> groupedRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>groupBy<span class="token punctuation">(</span>_<span class="token operator">%</span><span class="token number">2</span><span class="token punctuation">)</span>
    groupedRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>

    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  groupBy分组，按字母开头划分
 */</span>
<span class="token keyword">object</span> Spark06_RDD_Operator_groupBy_Transform1 <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - groupBy</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token string">"Hello"</span><span class="token punctuation">,</span> <span class="token string">"Spark"</span><span class="token punctuation">,</span> <span class="token string">"Scala"</span><span class="token punctuation">,</span> <span class="token string">"Hadoop"</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
    <span class="token comment" spellcheck="true">// 分组和分区没有必然的关系</span>
    <span class="token keyword">val</span> groupRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>groupBy<span class="token punctuation">(</span>_<span class="token punctuation">.</span>charAt<span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
    groupRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>
    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token keyword">import</span> java<span class="token punctuation">.</span>text<span class="token punctuation">.</span>SimpleDateFormat

<span class="token comment" spellcheck="true">/*
  groupBy分组，按时间统计数量
 */</span>
<span class="token keyword">object</span> Spark06_RDD_Operator_groupBy_Transform_Test <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>

    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 转换算子 - groupBy</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>textFile<span class="token punctuation">(</span><span class="token string">"datas/apache.log"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> timeRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>map<span class="token punctuation">(</span>
      line <span class="token keyword">=></span> <span class="token punctuation">{</span>
        <span class="token keyword">val</span> datas <span class="token operator">=</span> line<span class="token punctuation">.</span>split<span class="token punctuation">(</span><span class="token string">" "</span><span class="token punctuation">)</span>
        <span class="token keyword">val</span> time <span class="token operator">=</span> datas<span class="token punctuation">(</span><span class="token number">3</span><span class="token punctuation">)</span>
        <span class="token keyword">val</span> sdf <span class="token operator">=</span> <span class="token keyword">new</span> SimpleDateFormat<span class="token punctuation">(</span><span class="token string">"dd/MM/yyyy:HH:mm:ss"</span><span class="token punctuation">)</span>
        <span class="token keyword">val</span> date <span class="token operator">=</span> sdf<span class="token punctuation">.</span>parse<span class="token punctuation">(</span>time<span class="token punctuation">)</span>

        <span class="token keyword">val</span> sdf1 <span class="token operator">=</span> <span class="token keyword">new</span> SimpleDateFormat<span class="token punctuation">(</span><span class="token string">"dd"</span><span class="token punctuation">)</span>
        <span class="token keyword">val</span> day <span class="token operator">=</span> sdf1<span class="token punctuation">.</span>format<span class="token punctuation">(</span>date<span class="token punctuation">)</span>
        <span class="token punctuation">(</span>day<span class="token punctuation">,</span> <span class="token number">1</span><span class="token punctuation">)</span>
      <span class="token punctuation">}</span>
    <span class="token punctuation">)</span><span class="token punctuation">.</span>groupBy<span class="token punctuation">(</span>_<span class="token punctuation">.</span>_1<span class="token punctuation">)</span>

    timeRDD<span class="token punctuation">.</span>map<span class="token punctuation">{</span>
      <span class="token keyword">case</span> <span class="token punctuation">(</span>day<span class="token punctuation">,</span> iter<span class="token punctuation">)</span> <span class="token keyword">=></span>
        <span class="token punctuation">(</span>day<span class="token punctuation">,</span> iter<span class="token punctuation">.</span>size<span class="token punctuation">)</span>
    <span class="token punctuation">}</span><span class="token punctuation">.</span>collect<span class="token punctuation">.</span>sortBy<span class="token punctuation">(</span>_<span class="token punctuation">.</span>_2<span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>

<span class="token comment" spellcheck="true">//    val timeRDD = rdd.map(_.split(" ")(3))</span>
<span class="token comment" spellcheck="true">//    val day = timeRDD.map(_.split("/")(0)).map((_,1)).reduceByKey(_+_)</span>
<span class="token comment" spellcheck="true">//    day.collect().foreach(println)</span>
    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>

<span class="token punctuation">}</span>
</code></pre>
<h5 id="7-filter"><a href="#7-filter" class="headerlink" title="7.filter"></a>7.filter</h5><blockquote>
<p>将数据根据指定的规则进行筛选过滤，符合规则的数据保留，不符合规则的数据丢弃。当数据进行筛选过滤后，分区不变，但是分区内的数据可能不均衡，生产环境下，可能会出现数据倾斜。</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  filter 从服务器日志数据 apache.log 中获取 2015 年 5 月 17 日的请求路径
 */</span>
<span class="token keyword">object</span> Spark07_RDD_Operator_filter_Transform_Test <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>
    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 算子 - filter</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>textFile<span class="token punctuation">(</span><span class="token string">"datas/apache.log"</span><span class="token punctuation">)</span>

    rdd<span class="token punctuation">.</span>filter<span class="token punctuation">(</span>
      line <span class="token keyword">=></span> <span class="token punctuation">{</span>
        <span class="token keyword">val</span> data <span class="token operator">=</span> line<span class="token punctuation">.</span>split<span class="token punctuation">(</span><span class="token string">" "</span><span class="token punctuation">)</span>
        <span class="token keyword">val</span> time <span class="token operator">=</span> data<span class="token punctuation">(</span><span class="token number">3</span><span class="token punctuation">)</span>
        time<span class="token punctuation">.</span>startsWith<span class="token punctuation">(</span><span class="token string">"17/05/2015"</span><span class="token punctuation">)</span>
      <span class="token punctuation">}</span>
    <span class="token punctuation">)</span><span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>

    sc<span class="token punctuation">.</span>stop<span class="token punctuation">(</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>
<span class="token punctuation">}</span>
</code></pre>
<h5 id="8-sample"><a href="#8-sample" class="headerlink" title="8.sample"></a>8.sample</h5><blockquote>
<p>根据指定的规则从数据集中抽取数据</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">val</span> dataRDD <span class="token operator">=</span> sparkContext<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span>
 <span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span>
<span class="token punctuation">)</span><span class="token punctuation">,</span><span class="token number">1</span><span class="token punctuation">)</span>
<span class="token comment" spellcheck="true">// 抽取数据不放回（伯努利算法）</span>
<span class="token comment" spellcheck="true">// 伯努利算法：又叫 0、1 分布。例如扔硬币，要么正面，要么反面。</span>
<span class="token comment" spellcheck="true">// 具体实现：根据种子和随机算法算出一个数和第二个参数设置几率比较，小于第二个参数要，大于不</span>
要
<span class="token comment" spellcheck="true">// 第一个参数：抽取的数据是否放回，false：不放回</span>
<span class="token comment" spellcheck="true">// 第二个参数：抽取的几率，范围在[0,1]之间,0：全不取；1：全取；</span>
<span class="token comment" spellcheck="true">// 第三个参数：随机数种子</span>
<span class="token keyword">val</span> dataRDD1 <span class="token operator">=</span> dataRDD<span class="token punctuation">.</span>sample<span class="token punctuation">(</span><span class="token boolean">false</span><span class="token punctuation">,</span> <span class="token number">0.5</span><span class="token punctuation">)</span>
<span class="token comment" spellcheck="true">// 抽取数据放回（泊松算法）</span>
<span class="token comment" spellcheck="true">// 第一个参数：抽取的数据是否放回，true：放回；false：不放回</span>
<span class="token comment" spellcheck="true">// 第二个参数：重复数据的几率，范围大于等于 0.表示每一个元素被期望抽取到的次数</span>
<span class="token comment" spellcheck="true">// 第三个参数：随机数种子</span>
<span class="token keyword">val</span> dataRDD2 <span class="token operator">=</span> dataRDD<span class="token punctuation">.</span>sample<span class="token punctuation">(</span><span class="token boolean">true</span><span class="token punctuation">,</span> <span class="token number">2</span><span class="token punctuation">)</span>
</code></pre>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  sample
 */</span>
<span class="token keyword">object</span> Spark08_RDD_Operator_sample_Transform <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>
    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 算子 - sample</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span><span class="token punctuation">,</span><span class="token number">5</span><span class="token punctuation">,</span><span class="token number">6</span><span class="token punctuation">,</span><span class="token number">7</span><span class="token punctuation">,</span><span class="token number">8</span><span class="token punctuation">,</span><span class="token number">9</span><span class="token punctuation">,</span><span class="token number">10</span><span class="token punctuation">)</span><span class="token punctuation">)</span>


    <span class="token comment" spellcheck="true">/*
      sample算子需要传递三个参数
      1.第一个参数表示，抽取数据后是否将数据返回 true（放回），false（丢弃）
      2.第二个参数表示，数据源中每条数据被抽取的概率
      基准值
      3.第三个参数表示，抽取数据时随机算法的种子
      如果不传递第三个参数，那么使用的就是当前系统时间
     */</span>
<span class="token comment" spellcheck="true">//    println(rdd.sample(</span>
<span class="token comment" spellcheck="true">//      false,</span>
<span class="token comment" spellcheck="true">//      0.5,</span>
<span class="token comment" spellcheck="true">//      1</span>
<span class="token comment" spellcheck="true">//    ).collect().mkString(","))</span>
    println<span class="token punctuation">(</span>rdd<span class="token punctuation">.</span>sample<span class="token punctuation">(</span>
      <span class="token boolean">true</span><span class="token punctuation">,</span>
      <span class="token number">0.5</span><span class="token punctuation">,</span>
      <span class="token number">1</span>
    <span class="token punctuation">)</span><span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>mkString<span class="token punctuation">(</span><span class="token string">","</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
  <span class="token punctuation">}</span>
<span class="token punctuation">}</span>
</code></pre>
<h5 id="9-distinct"><a href="#9-distinct" class="headerlink" title="9.distinct"></a>9.distinct</h5><blockquote>
<p>将数据集中重复的数据去重</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  distinct 去重
 */</span>
<span class="token keyword">object</span> Spark09_RDD_Operator_distinct_Transform <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>
    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 算子 - distinct</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span><span class="token punctuation">,</span><span class="token number">5</span><span class="token punctuation">,</span><span class="token number">5</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">,</span><span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> rdd1 <span class="token operator">=</span> rdd<span class="token punctuation">.</span>distinct<span class="token punctuation">(</span><span class="token punctuation">)</span>
    rdd1<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>
  <span class="token punctuation">}</span>
<span class="token punctuation">}</span>
</code></pre>
<h5 id="10-coalesce"><a href="#10-coalesce" class="headerlink" title="10.coalesce"></a>10.coalesce</h5><blockquote>
<p>根据数据量缩减分区，用于大数据集过滤后，提高小数据集的执行效率。当 spark 程序中，存在过多的小任务的时候，可以通过 coalesce 方法，收缩合并分区，减少分区的个数，减小任务调度成本</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  coalesce 重新定义分区数，一般用来缩减分区
 */</span>
<span class="token keyword">object</span> Spark10_RDD_Operator_coalesce_Transform <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>
    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 算子 - coalesce</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span><span class="token punctuation">,</span><span class="token number">5</span><span class="token punctuation">,</span><span class="token number">6</span><span class="token punctuation">)</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// coalesce 默认情况下不会将分区的数据打乱重新组合</span>
    <span class="token comment" spellcheck="true">// 这种情况下的缩减分区可能会导致数据不均衡，出现数据倾斜</span>
    <span class="token comment" spellcheck="true">// 如果想要让数据均衡，可以进行shuffle处理</span>
    <span class="token comment" spellcheck="true">// coalesce 算子可以扩大分区，但必须进行shuffle操作，否则不起作用</span>
<span class="token comment" spellcheck="true">//    val rdd1 = rdd.coalesce(2)</span>
    <span class="token keyword">val</span> rdd1 <span class="token operator">=</span> rdd<span class="token punctuation">.</span>coalesce<span class="token punctuation">(</span><span class="token number">3</span><span class="token punctuation">,</span> <span class="token boolean">true</span><span class="token punctuation">)</span>
    rdd1<span class="token punctuation">.</span>saveAsTextFile<span class="token punctuation">(</span><span class="token string">"output"</span><span class="token punctuation">)</span>

  <span class="token punctuation">}</span>
<span class="token punctuation">}</span>
</code></pre>
<h5 id="11-repartition"><a href="#11-repartition" class="headerlink" title="11.repartition"></a>11.repartition</h5><blockquote>
<p>该操作内部其实执行的是 coalesce 操作，参数 shuffle 的默认值为 true。无论是将分区数多的RDD 转换为分区数少的 RDD，还是将分区数少的 RDD 转换为分区数多的 RDD，repartition操作都可以完成，因为无论如何都会经 shuffle 过程。</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  repartition 重新定义分区数，实际底层调用 coalesce
 */</span>
<span class="token keyword">object</span> Spark11_RDD_Operator_repartition_Transform <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>
    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 算子 - repartition</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">,</span><span class="token number">4</span><span class="token punctuation">,</span><span class="token number">5</span><span class="token punctuation">,</span><span class="token number">6</span><span class="token punctuation">)</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">)</span>

    <span class="token keyword">val</span> rdd1 <span class="token operator">=</span> rdd<span class="token punctuation">.</span>repartition<span class="token punctuation">(</span><span class="token number">3</span><span class="token punctuation">)</span>
    rdd1<span class="token punctuation">.</span>saveAsTextFile<span class="token punctuation">(</span><span class="token string">"output"</span><span class="token punctuation">)</span>

  <span class="token punctuation">}</span>
<span class="token punctuation">}</span>
</code></pre>
<h5 id="12-sortBy"><a href="#12-sortBy" class="headerlink" title="12.sortBy"></a>12.sortBy</h5><blockquote>
<p>该操作用于排序数据。在排序之前，可以将数据通过 f 函数进行处理，之后按照 f 函数处理的结果进行排序，默认为升序排列。排序后新产生的 RDD 的分区数与原 RDD 的分区数一致。中间存在 shuffle 的过程</p>
</blockquote>
<pre class=" language-scala"><code class="language-scala"><span class="token keyword">package</span> com<span class="token punctuation">.</span>atguigu<span class="token punctuation">.</span>bigdata<span class="token punctuation">.</span>spark<span class="token punctuation">.</span>core<span class="token punctuation">.</span>rdd<span class="token punctuation">.</span>operator<span class="token punctuation">.</span>transform

<span class="token keyword">import</span> org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>spark<span class="token punctuation">.</span><span class="token punctuation">{</span>SparkConf<span class="token punctuation">,</span> SparkContext<span class="token punctuation">}</span>

<span class="token comment" spellcheck="true">/*
  sortBy
 */</span>
<span class="token keyword">object</span> Spark12_RDD_Operator_sortBy_Transform1 <span class="token punctuation">{</span>
  <span class="token keyword">def</span> main<span class="token punctuation">(</span>args<span class="token operator">:</span> Array<span class="token punctuation">[</span><span class="token builtin">String</span><span class="token punctuation">]</span><span class="token punctuation">)</span><span class="token operator">:</span> <span class="token builtin">Unit</span> <span class="token operator">=</span> <span class="token punctuation">{</span>
    <span class="token keyword">val</span> sparkConf <span class="token operator">=</span> <span class="token keyword">new</span> SparkConf<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setMaster<span class="token punctuation">(</span><span class="token string">"local[*]"</span><span class="token punctuation">)</span><span class="token punctuation">.</span>setAppName<span class="token punctuation">(</span><span class="token string">"Operator"</span><span class="token punctuation">)</span>
    <span class="token keyword">val</span> sc <span class="token operator">=</span> <span class="token keyword">new</span> SparkContext<span class="token punctuation">(</span>sparkConf<span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// TODO 算子 - sortBy</span>
    <span class="token keyword">val</span> rdd <span class="token operator">=</span> sc<span class="token punctuation">.</span>makeRDD<span class="token punctuation">(</span>List<span class="token punctuation">(</span><span class="token punctuation">(</span><span class="token string">"1"</span><span class="token punctuation">,</span><span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">,</span><span class="token punctuation">(</span><span class="token string">"11"</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">)</span><span class="token punctuation">,</span><span class="token punctuation">(</span><span class="token string">"2"</span><span class="token punctuation">,</span><span class="token number">3</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">,</span><span class="token number">2</span><span class="token punctuation">)</span>

    <span class="token comment" spellcheck="true">// sortBy方法可以根据指定的规则对数据源中的数据进行排序，默认为升序，第二个参数可以改变排序的方式</span>
    <span class="token keyword">val</span> newRDD <span class="token operator">=</span> rdd<span class="token punctuation">.</span>sortBy<span class="token punctuation">(</span>x<span class="token keyword">=></span>x<span class="token punctuation">.</span>_1<span class="token punctuation">.</span>toInt<span class="token punctuation">,</span> <span class="token boolean">false</span><span class="token punctuation">)</span>
    newRDD<span class="token punctuation">.</span>collect<span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span>foreach<span class="token punctuation">(</span>println<span class="token punctuation">)</span>

  <span class="token punctuation">}</span>
<span class="token punctuation">}</span>
</code></pre>

                
            </div>
            <hr/>

            

    <div class="reprint" id="reprint-statement">
        
            <div class="reprint__author">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-user">
                        文章作者:
                    </i>
                </span>
                <span class="reprint-info">
                    <a href="/about" rel="external nofollow noreferrer">Aunean</a>
                </span>
            </div>
            <div class="reprint__type">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-link">
                        文章链接:
                    </i>
                </span>
                <span class="reprint-info">
                    <a href="https://www.shiyiri.top/node/10.html">https://www.shiyiri.top/node/10.html</a>
                </span>
            </div>
            <div class="reprint__notice">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-copyright">
                        版权声明:
                    </i>
                </span>
                <span class="reprint-info">
                    本博客所有文章除特別声明外，均采用
                    <a href="https://creativecommons.org/licenses/by/4.0/deed.zh" rel="external nofollow noreferrer" target="_blank">CC BY 4.0</a>
                    许可协议。转载请注明来源
                    <a href="/about" target="_blank">Aunean</a>
                    !
                </span>
            </div>
        
    </div>

    <script async defer>
      document.addEventListener("copy", function (e) {
        let toastHTML = '<span>复制成功，请遵循本文的转载规则</span><button class="btn-flat toast-action" onclick="navToReprintStatement()" style="font-size: smaller">查看</a>';
        M.toast({html: toastHTML})
      });

      function navToReprintStatement() {
        $("html, body").animate({scrollTop: $("#reprint-statement").offset().top - 80}, 800);
      }
    </script>



            <div class="tag_share" style="display: block;">
                <div class="post-meta__tag-list" style="display: inline-block;">
                    
                        <div class="article-tag">
                            
                                <a href="/tags/Spark/">
                                    <span class="chip bg-color">Spark</span>
                                </a>
                            
                        </div>
                    
                </div>
                <div class="post_share" style="zoom: 80%; width: fit-content; display: inline-block; float: right; margin: -0.15rem 0;">
                    <link rel="stylesheet" type="text/css" href="/libs/share/css/share.min.css">
<div id="article-share">

    
    <div class="social-share" data-sites="twitter,facebook,google,qq,qzone,wechat,weibo,douban,linkedin" data-wechat-qrcode-helper="<p>微信扫一扫即可分享！</p>"></div>
    <script src="/libs/share/js/social-share.min.js"></script>
    

    

</div>

                </div>
            </div>
            
                <style>
    #reward {
        margin: 40px 0;
        text-align: center;
    }

    #reward .reward-link {
        font-size: 1.4rem;
        line-height: 38px;
    }

    #reward .btn-floating:hover {
        box-shadow: 0 6px 12px rgba(0, 0, 0, 0.2), 0 5px 15px rgba(0, 0, 0, 0.2);
    }

    #rewardModal {
        width: 320px;
        height: 350px;
    }

    #rewardModal .reward-title {
        margin: 15px auto;
        padding-bottom: 5px;
    }

    #rewardModal .modal-content {
        padding: 10px;
    }

    #rewardModal .close {
        position: absolute;
        right: 15px;
        top: 15px;
        color: rgba(0, 0, 0, 0.5);
        font-size: 1.3rem;
        line-height: 20px;
        cursor: pointer;
    }

    #rewardModal .close:hover {
        color: #ef5350;
        transform: scale(1.3);
        -moz-transform:scale(1.3);
        -webkit-transform:scale(1.3);
        -o-transform:scale(1.3);
    }

    #rewardModal .reward-tabs {
        margin: 0 auto;
        width: 210px;
    }

    .reward-tabs .tabs {
        height: 38px;
        margin: 10px auto;
        padding-left: 0;
    }

    .reward-content ul {
        padding-left: 0 !important;
    }

    .reward-tabs .tabs .tab {
        height: 38px;
        line-height: 38px;
    }

    .reward-tabs .tab a {
        color: #fff;
        background-color: #ccc;
    }

    .reward-tabs .tab a:hover {
        background-color: #ccc;
        color: #fff;
    }

    .reward-tabs .wechat-tab .active {
        color: #fff !important;
        background-color: #22AB38 !important;
    }

    .reward-tabs .alipay-tab .active {
        color: #fff !important;
        background-color: #019FE8 !important;
    }

    .reward-tabs .reward-img {
        width: 210px;
        height: 210px;
    }
</style>

<div id="reward">
    <a href="#rewardModal" class="reward-link modal-trigger btn-floating btn-medium waves-effect waves-light red">赏</a>

    <!-- Modal Structure -->
    <div id="rewardModal" class="modal">
        <div class="modal-content">
            <a class="close modal-close"><i class="fas fa-times"></i></a>
            <h4 class="reward-title">你的赏识是我前进的动力</h4>
            <div class="reward-content">
                <div class="reward-tabs">
                    <ul class="tabs row">
                        <li class="tab col s6 alipay-tab waves-effect waves-light"><a href="#alipay">支付宝</a></li>
                        <li class="tab col s6 wechat-tab waves-effect waves-light"><a href="#wechat">微 信</a></li>
                    </ul>
                    <div id="alipay">
                        <img src="/medias/reward/alipay.jpg" class="reward-img" alt="支付宝打赏二维码">
                    </div>
                    <div id="wechat">
                        <img src="/medias/reward/wechat.png" class="reward-img" alt="微信打赏二维码">
                    </div>
                </div>
            </div>
        </div>
    </div>
</div>

<script>
    $(function () {
        $('.tabs').tabs();
    });
</script>

            
        </div>
    </div>

    
        <link rel="stylesheet" href="/libs/gitalk/gitalk.css">
<link rel="stylesheet" href="/css/my-gitalk.css">

<div class="card gitalk-card" data-aos="fade-up">
    <div class="comment_headling" style="font-size: 20px; font-weight: 700; position: relative; padding-left: 20px; top: 15px; padding-bottom: 5px;">
        <i class="fas fa-comments fa-fw" aria-hidden="true"></i>
        <span>评论</span>
    </div>
    <div id="gitalk-container" class="card-content"></div>
</div>

<script src="/libs/gitalk/gitalk.min.js"></script>
<script>
    let gitalk = new Gitalk({
        clientID: '43918ae8f8557340e530',
        clientSecret: '016469b0d80284ccd95b27ea68909008a765d99e',
        repo: 'comment_by_blog',
        owner: 'Aunean-ls',
        admin: "Aunean-ls",
        id: '2021-05-09T12-08-00',
        distractionFreeMode: false  // Facebook-like distraction free mode
    });

    gitalk.render('gitalk-container');
</script>

    

    

    

    

    

    

    

    

<article id="prenext-posts" class="prev-next articles">
    <div class="row article-row">
        
        <div class="article col s12 m6" data-aos="fade-up">
            <div class="article-badge left-badge text-color">
                <i class="fas fa-chevron-left"></i>&nbsp;上一篇</div>
            <div class="card">
                <a href="/node/100.html">
                    <div class="card-image">
                        
                        
                        <img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/u002.webp" class="responsive-img" alt="猫眼字体处理">
                        
                        <span class="card-title">猫眼字体处理</span>
                    </div>
                </a>
                <div class="card-content article-content">
                    <div class="summary block-with-text">
                        
                            本文记录对猫眼字体的解析和采集
                        
                    </div>
                    <div class="publish-info">
                        <span class="publish-date">
                            <i class="far fa-clock fa-fw icon-date"></i>2021-05-31
                        </span>
                        <span class="publish-author">
                            
                            <i class="fas fa-bookmark fa-fw icon-category"></i>
                            
                            <a href="/categories/python/" class="post-category">
                                    python
                                </a>
                            
                            
                        </span>
                    </div>
                </div>
                
                <div class="card-action article-tags">
                    
                    <a href="/tags/%E6%95%B0%E6%8D%AE%E9%87%87%E9%9B%86/">
                        <span class="chip bg-color">数据采集</span>
                    </a>
                    
                </div>
                
            </div>
        </div>
        
        
        <div class="article col s12 m6" data-aos="fade-up">
            <div class="article-badge right-badge text-color">
                下一篇&nbsp;<i class="fas fa-chevron-right"></i>
            </div>
            <div class="card">
                <a href="/node/8.html">
                    <div class="card-image">
                        
                        
                        <img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/u024.webp" class="responsive-img" alt="Scala笔记">
                        
                        <span class="card-title">Scala笔记</span>
                    </div>
                </a>
                <div class="card-content article-content">
                    <div class="summary block-with-text">
                        
                            本文记录Scala学习时的一些笔记
                        
                    </div>
                    <div class="publish-info">
                            <span class="publish-date">
                                <i class="far fa-clock fa-fw icon-date"></i>2021-05-04
                            </span>
                        <span class="publish-author">
                            
                            <i class="fas fa-bookmark fa-fw icon-category"></i>
                            
                            <a href="/categories/%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/" class="post-category">
                                    学习笔记
                                </a>
                            
                            
                        </span>
                    </div>
                </div>
                
                <div class="card-action article-tags">
                    
                    <a href="/tags/Scala/">
                        <span class="chip bg-color">Scala</span>
                    </a>
                    
                </div>
                
            </div>
        </div>
        
    </div>
</article>

</div>



<!-- 代码块功能依赖 -->
<script type="text/javascript" src="/libs/codeBlock/codeBlockFuction.js"></script>

<!-- 代码语言 -->

<script type="text/javascript" src="/libs/codeBlock/codeLang.js"></script>


<!-- 代码块复制 -->

<script type="text/javascript" src="/libs/codeBlock/codeCopy.js"></script>


<!-- 代码块收缩 -->

<script type="text/javascript" src="/libs/codeBlock/codeShrink.js"></script>


    </div>
    <div id="toc-aside" class="expanded col l3 hide-on-med-and-down">
        <div class="toc-widget card" style="background-color: white;">
            <div class="toc-title"><i class="far fa-list-alt"></i>&nbsp;&nbsp;目录</div>
            <div id="toc-content"></div>
        </div>
    </div>
</div>

<!-- TOC 悬浮按钮. -->

<div id="floating-toc-btn" class="hide-on-med-and-down">
    <a class="btn-floating btn-large bg-color">
        <i class="fas fa-list-ul"></i>
    </a>
</div>


<script src="/libs/tocbot/tocbot.min.js"></script>
<script>
    $(function () {
        tocbot.init({
            tocSelector: '#toc-content',
            contentSelector: '#articleContent',
            headingsOffset: -($(window).height() * 0.4 - 45),
            collapseDepth: Number('0'),
            headingSelector: 'h2, h3, h4, h5, h6'
        });

        // modify the toc link href to support Chinese.
        let i = 0;
        let tocHeading = 'toc-heading-';
        $('#toc-content a').each(function () {
            $(this).attr('href', '#' + tocHeading + (++i));
        });

        // modify the heading title id to support Chinese.
        i = 0;
        $('#articleContent').children('h2, h3, h4, h5, h6').each(function () {
            $(this).attr('id', tocHeading + (++i));
        });

        // Set scroll toc fixed.
        let tocHeight = parseInt($(window).height() * 0.4 - 64);
        let $tocWidget = $('.toc-widget');
        $(window).scroll(function () {
            let scroll = $(window).scrollTop();
            /* add post toc fixed. */
            if (scroll > tocHeight) {
                $tocWidget.addClass('toc-fixed');
            } else {
                $tocWidget.removeClass('toc-fixed');
            }
        });

        
        /* 修复文章卡片 div 的宽度. */
        let fixPostCardWidth = function (srcId, targetId) {
            let srcDiv = $('#' + srcId);
            if (srcDiv.length === 0) {
                return;
            }

            let w = srcDiv.width();
            if (w >= 450) {
                w = w + 21;
            } else if (w >= 350 && w < 450) {
                w = w + 18;
            } else if (w >= 300 && w < 350) {
                w = w + 16;
            } else {
                w = w + 14;
            }
            $('#' + targetId).width(w);
        };

        // 切换TOC目录展开收缩的相关操作.
        const expandedClass = 'expanded';
        let $tocAside = $('#toc-aside');
        let $mainContent = $('#main-content');
        $('#floating-toc-btn .btn-floating').click(function () {
            if ($tocAside.hasClass(expandedClass)) {
                $tocAside.removeClass(expandedClass).hide();
                $mainContent.removeClass('l9');
            } else {
                $tocAside.addClass(expandedClass).show();
                $mainContent.addClass('l9');
            }
            fixPostCardWidth('artDetail', 'prenext-posts');
        });
        
    });
</script>

    

</main>




    <footer class="page-footer bg-color">
    
        <link rel="stylesheet" href="/libs/aplayer/APlayer.min.css">
<style>
    .aplayer .aplayer-lrc p {
        
        display: none;
        
        font-size: 12px;
        font-weight: 700;
        line-height: 16px !important;
    }

    .aplayer .aplayer-lrc p.aplayer-lrc-current {
        
        display: none;
        
        font-size: 15px;
        color: #42b983;
    }

    
    .aplayer.aplayer-fixed.aplayer-narrow .aplayer-body {
        left: -66px !important;
    }

    .aplayer.aplayer-fixed.aplayer-narrow .aplayer-body:hover {
        left: 0px !important;
    }

    
</style>
<div class="">
    
    <div class="row">
        <meting-js class="col l8 offset-l2 m10 offset-m1 s12"
                   server="tencent"
                   type="playlist"
                   id="4628814494"
                   fixed='true'
                   autoplay='false'
                   theme='#42b983'
                   loop='all'
                   order='random'
                   preload='auto'
                   volume='0.7'
                   list-folded='true'
        >
        </meting-js>
    </div>
</div>

<script src="/libs/aplayer/APlayer.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/meting@2/dist/Meting.min.js"></script>

    
    <div class="container row center-align" style="margin-bottom: 15px !important;">
        <div class="col s12 m8 l8 copy-right">
            Copyright&nbsp;&copy;
            
                <span id="year">2021</span>
            
            <span id="year">2021</span>
            <a href="/about" target="_blank">Aunean</a>
            |&nbsp;Powered by&nbsp;<a href="https://hexo.io/" target="_blank">Hexo</a>
            |&nbsp;Theme&nbsp;<a href="https://github.com/blinkfox/hexo-theme-matery" target="_blank">Matery</a>
            <br>
            
            &nbsp;<i class="fas fa-chart-area"></i>&nbsp;站点总字数:&nbsp;<span
                class="white-color">64.8k</span>&nbsp;字
            
            
            
            
            
            
            <span id="busuanzi_container_site_pv">
                |&nbsp;<i class="far fa-eye"></i>&nbsp;总访问量:&nbsp;<span id="busuanzi_value_site_pv"
                    class="white-color"></span>&nbsp;次
            </span>
            
            
            <span id="busuanzi_container_site_uv">
                |&nbsp;<i class="fas fa-users"></i>&nbsp;总访问人数:&nbsp;<span id="busuanzi_value_site_uv"
                    class="white-color"></span>&nbsp;人
            </span>
            
            <br>
            
            <span id="sitetime">载入运行时间...</span>
            <script>
                function siteTime() {
                    var seconds = 1000;
                    var minutes = seconds * 60;
                    var hours = minutes * 60;
                    var days = hours * 24;
                    var years = days * 365;
                    var today = new Date();
                    var startYear = "2021";
                    var startMonth = "4";
                    var startDate = "11";
                    var startHour = "0";
                    var startMinute = "0";
                    var startSecond = "0";
                    var todayYear = today.getFullYear();
                    var todayMonth = today.getMonth() + 1;
                    var todayDate = today.getDate();
                    var todayHour = today.getHours();
                    var todayMinute = today.getMinutes();
                    var todaySecond = today.getSeconds();
                    var t1 = Date.UTC(startYear, startMonth, startDate, startHour, startMinute, startSecond);
                    var t2 = Date.UTC(todayYear, todayMonth, todayDate, todayHour, todayMinute, todaySecond);
                    var diff = t2 - t1;
                    var diffYears = Math.floor(diff / years);
                    var diffDays = Math.floor((diff / days) - diffYears * 365);
                    var diffHours = Math.floor((diff - (diffYears * 365 + diffDays) * days) / hours);
                    var diffMinutes = Math.floor((diff - (diffYears * 365 + diffDays) * days - diffHours * hours) /
                        minutes);
                    var diffSeconds = Math.floor((diff - (diffYears * 365 + diffDays) * days - diffHours * hours -
                        diffMinutes * minutes) / seconds);
                    if (startYear == todayYear) {
                        document.getElementById("year").innerHTML = todayYear;
                        document.getElementById("sitetime").innerHTML = "本站已安全运行 " + diffDays + " 天 " + diffHours +
                            " 小时 " + diffMinutes + " 分钟 " + diffSeconds + " 秒";
                    } else {
                        document.getElementById("year").innerHTML = startYear + " - " + todayYear;
                        document.getElementById("sitetime").innerHTML = "本站已安全运行 " + diffYears + " 年 " + diffDays +
                            " 天 " + diffHours + " 小时 " + diffMinutes + " 分钟 " + diffSeconds + " 秒";
                    }
                }
                setInterval(siteTime, 1000);
            </script>
            
            <br>
            
        </div>
        <div class="col s12 m4 l4 social-link social-statis">
    <a href="https://github.com/Aunean-ls" class="tooltipped" target="_blank" data-tooltip="访问我的GitHub" data-position="top" data-delay="50">
        <i class="fab fa-github"></i>
    </a>



    <a href="mailto:1453357375@qq.com" class="tooltipped" target="_blank" data-tooltip="邮件联系我" data-position="top" data-delay="50">
        <i class="fas fa-envelope-open"></i>
    </a>







    <a href="tencent://AddContact/?fromId=50&fromSubId=1&subcmd=all&uin=1453357375" class="tooltipped" target="_blank" data-tooltip="QQ联系我: 1453357375" data-position="top" data-delay="50">
        <i class="fab fa-qq"></i>
    </a>







    <a href="/atom.xml" class="tooltipped" target="_blank" data-tooltip="RSS 订阅" data-position="top" data-delay="50">
        <i class="fas fa-rss"></i>
    </a>

</div>
    </div>
</footer>

<div class="progress-bar"></div>


    <!-- 搜索遮罩框 -->
<div id="searchModal" class="modal">
    <div class="modal-content">
        <div class="search-header">
            <span class="title"><i class="fas fa-search"></i>&nbsp;&nbsp;搜索</span>
            <input type="search" id="searchInput" name="s" placeholder="请输入搜索的关键字"
                   class="search-input">
        </div>
        <div id="searchResult"></div>
    </div>
</div>

<script type="text/javascript">
$(function () {
    var searchFunc = function (path, search_id, content_id) {
        'use strict';
        $.ajax({
            url: path,
            dataType: "xml",
            success: function (xmlResponse) {
                // get the contents from search data
                var datas = $("entry", xmlResponse).map(function () {
                    return {
                        title: $("title", this).text(),
                        content: $("content", this).text(),
                        url: $("url", this).text()
                    };
                }).get();
                var $input = document.getElementById(search_id);
                var $resultContent = document.getElementById(content_id);
                $input.addEventListener('input', function () {
                    var str = '<ul class=\"search-result-list\">';
                    var keywords = this.value.trim().toLowerCase().split(/[\s\-]+/);
                    $resultContent.innerHTML = "";
                    if (this.value.trim().length <= 0) {
                        return;
                    }
                    // perform local searching
                    datas.forEach(function (data) {
                        var isMatch = true;
                        var data_title = data.title.trim().toLowerCase();
                        var data_content = data.content.trim().replace(/<[^>]+>/g, "").toLowerCase();
                        var data_url = data.url;
                        data_url = data_url.indexOf('/') === 0 ? data.url : '/' + data_url;
                        var index_title = -1;
                        var index_content = -1;
                        var first_occur = -1;
                        // only match artiles with not empty titles and contents
                        if (data_title !== '' && data_content !== '') {
                            keywords.forEach(function (keyword, i) {
                                index_title = data_title.indexOf(keyword);
                                index_content = data_content.indexOf(keyword);
                                if (index_title < 0 && index_content < 0) {
                                    isMatch = false;
                                } else {
                                    if (index_content < 0) {
                                        index_content = 0;
                                    }
                                    if (i === 0) {
                                        first_occur = index_content;
                                    }
                                }
                            });
                        }
                        // show search results
                        if (isMatch) {
                            str += "<li><a href='" + data_url + "' class='search-result-title'>" + data_title + "</a>";
                            var content = data.content.trim().replace(/<[^>]+>/g, "");
                            if (first_occur >= 0) {
                                // cut out 100 characters
                                var start = first_occur - 20;
                                var end = first_occur + 80;
                                if (start < 0) {
                                    start = 0;
                                }
                                if (start === 0) {
                                    end = 100;
                                }
                                if (end > content.length) {
                                    end = content.length;
                                }
                                var match_content = content.substr(start, end);
                                // highlight all keywords
                                keywords.forEach(function (keyword) {
                                    var regS = new RegExp(keyword, "gi");
                                    match_content = match_content.replace(regS, "<em class=\"search-keyword\">" + keyword + "</em>");
                                });

                                str += "<p class=\"search-result\">" + match_content + "...</p>"
                            }
                            str += "</li>";
                        }
                    });
                    str += "</ul>";
                    $resultContent.innerHTML = str;
                });
            }
        });
    };

    searchFunc('/search.xml', 'searchInput', 'searchResult');
});
</script>

    <!-- 回到顶部按钮 -->
<div id="backTop" class="top-scroll">
    <a class="btn-floating btn-large waves-effect waves-light" href="#!">
        <i class="fas fa-arrow-up"></i>
    </a>
</div>


    <script src="/libs/materialize/materialize.min.js"></script>
    <script src="/libs/masonry/masonry.pkgd.min.js"></script>
    <script src="/libs/aos/aos.js"></script>
    <script src="/libs/scrollprogress/scrollProgress.min.js"></script>
    <script src="/libs/lightGallery/js/lightgallery-all.min.js"></script>
    <script src="/js/matery.js"></script>

    <script type="text/javascript">
        var OriginTitile=document.title,st;
        document.addEventListener("visibilitychange",function(){
            document.hidden?(document.title="ヽ(●-`Д´-)ノ页面丢失了",clearTimeout(st)):(document.title="(Ő∀Ő3)ノ又好了哦！",st=setTimeout(function(){document.title=OriginTitile},3e3))
        })
    </script>


    <script type="text/javascript">
    //只在桌面版网页启用特效
        var windowWidth = $(window).width();
        if (windowWidth > 768) {
            document.write('<script type="text/javascript" src="/js/sakura.js"><\/script>');
        }
    </script>


    <!-- Baidu Analytics -->

    <!-- Baidu Push -->

<script>
    (function () {
        var bp = document.createElement('script');
        var curProtocol = window.location.protocol.split(':')[0];
        if (curProtocol === 'https') {
            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
        } else {
            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
        }
        var s = document.getElementsByTagName("script")[0];
        s.parentNode.insertBefore(bp, s);
    })();
</script>

    
    <script src="/libs/others/clicklove.js" async="async"></script>
    
    
    <script async src="/libs/others/busuanzi.pure.mini.js"></script>
    

    

    

	
    
    <script type="text/javascript" color="0,0,255"
        pointColor="0,0,255" opacity='0.7'
        zIndex="-1" count="99"
        src="/libs/background/canvas-nest.js"></script>
    

    
    
    <script type="text/javascript" size="150" alpha='0.6'
        zIndex="-1" src="/libs/background/ribbon-refresh.min.js" async="async"></script>
    

    
    <script type="text/javascript" src="/libs/background/ribbon-dynamic.js" async="async"></script>
    

    
    <script src="/libs/instantpage/instantpage.js" type="module"></script>
    


</body>

</html>
